logstash-input-elasticsearch 4.0.3 → 4.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +8 -1
- data/docs/index.asciidoc +220 -0
- data/lib/logstash/inputs/elasticsearch.rb +5 -5
- data/logstash-input-elasticsearch.gemspec +2 -2
- data/spec/inputs/elasticsearch_spec.rb +1 -1
- metadata +21 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3001f304354780e0da016525097045e030d34e32
|
|
4
|
+
data.tar.gz: fcc39900a7ecb6a858829076430e7a8d1945195e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 31efd5f40520e84401c016472d49244c009fb21745775022dde902dcd49e9e50acaf76937fe5fbc2c560119ad489b888d0ac5d74cca2d10191ba2104c2647e37
|
|
7
|
+
data.tar.gz: ec949ec019e0580515a6b7e28e3c7b0c0006e2322b1861a5372a7ee6fabf66f3072fdc0bbe235f52f8002a440b512c478061b14f82095451c743b597e1a5827f
|
data/Gemfile
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
source 'https://rubygems.org'
|
|
2
2
|
|
|
3
|
-
# Specify your gem's dependencies in logstash-mass_effect.gemspec
|
|
4
3
|
gemspec
|
|
4
|
+
|
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
|
7
|
+
|
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
:plugin: elasticsearch
|
|
2
|
+
:type: input
|
|
3
|
+
|
|
4
|
+
///////////////////////////////////////////
|
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
|
6
|
+
///////////////////////////////////////////
|
|
7
|
+
:version: %VERSION%
|
|
8
|
+
:release_date: %RELEASE_DATE%
|
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
|
11
|
+
///////////////////////////////////////////
|
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
|
13
|
+
///////////////////////////////////////////
|
|
14
|
+
|
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
|
16
|
+
|
|
17
|
+
=== Elasticsearch input plugin
|
|
18
|
+
|
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
|
20
|
+
|
|
21
|
+
==== Description
|
|
22
|
+
|
|
23
|
+
.Compatibility Note
|
|
24
|
+
[NOTE]
|
|
25
|
+
================================================================================
|
|
26
|
+
Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
|
|
27
|
+
called `http.content_type.required`. If this option is set to `true`, and you
|
|
28
|
+
are using Logstash 2.4 through 5.2, you need to update the Elasticsearch input
|
|
29
|
+
plugin to version 4.0.2 or higher.
|
|
30
|
+
|
|
31
|
+
================================================================================
|
|
32
|
+
|
|
33
|
+
Read from an Elasticsearch cluster, based on search query results.
|
|
34
|
+
This is useful for replaying test logs, reindexing, etc.
|
|
35
|
+
|
|
36
|
+
Example:
|
|
37
|
+
[source,ruby]
|
|
38
|
+
input {
|
|
39
|
+
# Read all documents from Elasticsearch matching the given query
|
|
40
|
+
elasticsearch {
|
|
41
|
+
hosts => "localhost"
|
|
42
|
+
query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }'
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
This would create an Elasticsearch query with the following format:
|
|
47
|
+
[source,json]
|
|
48
|
+
curl 'http://localhost:9200/logstash-*/_search?&scroll=1m&size=1000' -d '{
|
|
49
|
+
"query": {
|
|
50
|
+
"match": {
|
|
51
|
+
"statuscode": 200
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"sort": [ "_doc" ]
|
|
55
|
+
}'
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
[id="plugins-{type}s-{plugin}-options"]
|
|
59
|
+
==== Elasticsearch Input Configuration Options
|
|
60
|
+
|
|
61
|
+
This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
|
|
62
|
+
|
|
63
|
+
[cols="<,<,<",options="header",]
|
|
64
|
+
|=======================================================================
|
|
65
|
+
|Setting |Input type|Required
|
|
66
|
+
| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
|
|
67
|
+
| <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
|
|
68
|
+
| <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
|
|
69
|
+
| <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
|
|
70
|
+
| <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
|
|
71
|
+
| <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
|
|
72
|
+
| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
|
|
73
|
+
| <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
|
|
74
|
+
| <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
|
|
75
|
+
| <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
|
|
76
|
+
| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
|
|
77
|
+
| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
|
|
78
|
+
|=======================================================================
|
|
79
|
+
|
|
80
|
+
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
|
81
|
+
input plugins.
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
[id="plugins-{type}s-{plugin}-ca_file"]
|
|
86
|
+
===== `ca_file`
|
|
87
|
+
|
|
88
|
+
* Value type is <<path,path>>
|
|
89
|
+
* There is no default value for this setting.
|
|
90
|
+
|
|
91
|
+
SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
|
|
92
|
+
|
|
93
|
+
[id="plugins-{type}s-{plugin}-docinfo"]
|
|
94
|
+
===== `docinfo`
|
|
95
|
+
|
|
96
|
+
* Value type is <<boolean,boolean>>
|
|
97
|
+
* Default value is `false`
|
|
98
|
+
|
|
99
|
+
If set, include Elasticsearch document information such as index, type, and
|
|
100
|
+
the id in the event.
|
|
101
|
+
|
|
102
|
+
It might be important to note, with regards to metadata, that if you're
|
|
103
|
+
ingesting documents with the intent to re-index them (or just update them)
|
|
104
|
+
that the `action` option in the elasticsearch output wants to know how to
|
|
105
|
+
handle those things. It can be dynamically assigned with a field
|
|
106
|
+
added to the metadata.
|
|
107
|
+
|
|
108
|
+
Example
|
|
109
|
+
[source, ruby]
|
|
110
|
+
input {
|
|
111
|
+
elasticsearch {
|
|
112
|
+
hosts => "es.production.mysite.org"
|
|
113
|
+
index => "mydata-2018.09.*"
|
|
114
|
+
query => "*"
|
|
115
|
+
size => 500
|
|
116
|
+
scroll => "5m"
|
|
117
|
+
docinfo => true
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
output {
|
|
121
|
+
elasticsearch {
|
|
122
|
+
index => "copy-of-production.%{[@metadata][_index]}"
|
|
123
|
+
document_type => "%{[@metadata][_type]}"
|
|
124
|
+
document_id => "%{[@metadata][_id]}"
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
[id="plugins-{type}s-{plugin}-docinfo_fields"]
|
|
130
|
+
===== `docinfo_fields`
|
|
131
|
+
|
|
132
|
+
* Value type is <<array,array>>
|
|
133
|
+
* Default value is `["_index", "_type", "_id"]`
|
|
134
|
+
|
|
135
|
+
List of document metadata to move to the `docinfo_target` field
|
|
136
|
+
To learn more about Elasticsearch metadata fields read
|
|
137
|
+
http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html
|
|
138
|
+
|
|
139
|
+
[id="plugins-{type}s-{plugin}-docinfo_target"]
|
|
140
|
+
===== `docinfo_target`
|
|
141
|
+
|
|
142
|
+
* Value type is <<string,string>>
|
|
143
|
+
* Default value is `"@metadata"`
|
|
144
|
+
|
|
145
|
+
Where to move the Elasticsearch document information by default we use the @metadata field.
|
|
146
|
+
|
|
147
|
+
[id="plugins-{type}s-{plugin}-hosts"]
|
|
148
|
+
===== `hosts`
|
|
149
|
+
|
|
150
|
+
* Value type is <<array,array>>
|
|
151
|
+
* There is no default value for this setting.
|
|
152
|
+
|
|
153
|
+
List of elasticsearch hosts to use for querying.
|
|
154
|
+
each host can be either IP, HOST, IP:port or HOST:port
|
|
155
|
+
port defaults to 9200
|
|
156
|
+
|
|
157
|
+
[id="plugins-{type}s-{plugin}-index"]
|
|
158
|
+
===== `index`
|
|
159
|
+
|
|
160
|
+
* Value type is <<string,string>>
|
|
161
|
+
* Default value is `"logstash-*"`
|
|
162
|
+
|
|
163
|
+
The index or alias to search.
|
|
164
|
+
|
|
165
|
+
[id="plugins-{type}s-{plugin}-password"]
|
|
166
|
+
===== `password`
|
|
167
|
+
|
|
168
|
+
* Value type is <<password,password>>
|
|
169
|
+
* There is no default value for this setting.
|
|
170
|
+
|
|
171
|
+
Basic Auth - password
|
|
172
|
+
|
|
173
|
+
[id="plugins-{type}s-{plugin}-query"]
|
|
174
|
+
===== `query`
|
|
175
|
+
|
|
176
|
+
* Value type is <<string,string>>
|
|
177
|
+
* Default value is `"{ \"sort\": [ \"_doc\" ] }"`
|
|
178
|
+
|
|
179
|
+
The query to be executed. Read the Elasticsearch query DSL documentation
|
|
180
|
+
for more info
|
|
181
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
|
182
|
+
|
|
183
|
+
[id="plugins-{type}s-{plugin}-scroll"]
|
|
184
|
+
===== `scroll`
|
|
185
|
+
|
|
186
|
+
* Value type is <<string,string>>
|
|
187
|
+
* Default value is `"1m"`
|
|
188
|
+
|
|
189
|
+
This parameter controls the keepalive time in seconds of the scrolling
|
|
190
|
+
request and initiates the scrolling process. The timeout applies per
|
|
191
|
+
round trip (i.e. between the previous scroll request, to the next).
|
|
192
|
+
|
|
193
|
+
[id="plugins-{type}s-{plugin}-size"]
|
|
194
|
+
===== `size`
|
|
195
|
+
|
|
196
|
+
* Value type is <<number,number>>
|
|
197
|
+
* Default value is `1000`
|
|
198
|
+
|
|
199
|
+
This allows you to set the maximum number of hits returned per scroll.
|
|
200
|
+
|
|
201
|
+
[id="plugins-{type}s-{plugin}-ssl"]
|
|
202
|
+
===== `ssl`
|
|
203
|
+
|
|
204
|
+
* Value type is <<boolean,boolean>>
|
|
205
|
+
* Default value is `false`
|
|
206
|
+
|
|
207
|
+
SSL
|
|
208
|
+
|
|
209
|
+
[id="plugins-{type}s-{plugin}-user"]
|
|
210
|
+
===== `user`
|
|
211
|
+
|
|
212
|
+
* Value type is <<string,string>>
|
|
213
|
+
* There is no default value for this setting.
|
|
214
|
+
|
|
215
|
+
Basic Auth - username
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
[id="plugins-{type}s-{plugin}-common-options"]
|
|
220
|
+
include::{include_path}/{type}.asciidoc[]
|
|
@@ -43,8 +43,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
43
43
|
default :codec, "json"
|
|
44
44
|
|
|
45
45
|
# List of elasticsearch hosts to use for querying.
|
|
46
|
-
#
|
|
47
|
-
#
|
|
46
|
+
# Each host can be either IP, HOST, IP:port or HOST:port.
|
|
47
|
+
# Port defaults to 9200
|
|
48
48
|
config :hosts, :validate => :array
|
|
49
49
|
|
|
50
50
|
# The index or alias to search.
|
|
@@ -87,17 +87,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
87
87
|
# output {
|
|
88
88
|
# elasticsearch {
|
|
89
89
|
# index => "copy-of-production.%{[@metadata][_index]}"
|
|
90
|
-
#
|
|
90
|
+
# document_type => "%{[@metadata][_type]}"
|
|
91
91
|
# document_id => "%{[@metadata][_id]}"
|
|
92
92
|
# }
|
|
93
93
|
# }
|
|
94
94
|
#
|
|
95
95
|
config :docinfo, :validate => :boolean, :default => false
|
|
96
96
|
|
|
97
|
-
# Where to move the Elasticsearch document information
|
|
97
|
+
# Where to move the Elasticsearch document information. By default we use the @metadata field.
|
|
98
98
|
config :docinfo_target, :validate=> :string, :default => LogStash::Event::METADATA
|
|
99
99
|
|
|
100
|
-
# List of document metadata to move to the `docinfo_target` field
|
|
100
|
+
# List of document metadata to move to the `docinfo_target` field.
|
|
101
101
|
# To learn more about Elasticsearch metadata fields read
|
|
102
102
|
# http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html
|
|
103
103
|
config :docinfo_fields, :validate => :array, :default => ['_index', '_type', '_id']
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
|
4
|
-
s.version = '4.0.
|
|
4
|
+
s.version = '4.0.4'
|
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
|
6
6
|
s.summary = "Read from an Elasticsearch cluster, based on search query results"
|
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
|
|
|
11
11
|
s.require_paths = ["lib"]
|
|
12
12
|
|
|
13
13
|
# Files
|
|
14
|
-
s.files = Dir[
|
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
|
15
15
|
|
|
16
16
|
# Tests
|
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
|
@@ -168,7 +168,7 @@ describe LogStash::Inputs::Elasticsearch do
|
|
|
168
168
|
}
|
|
169
169
|
]
|
|
170
170
|
|
|
171
|
-
pipeline =
|
|
171
|
+
pipeline = new_pipeline_from_string(config_metadata_with_string)
|
|
172
172
|
queue = Queue.new
|
|
173
173
|
pipeline.instance_eval do
|
|
174
174
|
@output_func = lambda { |event| queue << event }
|
metadata
CHANGED
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-input-elasticsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.0.
|
|
4
|
+
version: 4.0.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
|
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
15
|
requirements:
|
|
17
16
|
- - ">="
|
|
18
17
|
- !ruby/object:Gem::Version
|
|
@@ -20,7 +19,10 @@ dependencies:
|
|
|
20
19
|
- - "<="
|
|
21
20
|
- !ruby/object:Gem::Version
|
|
22
21
|
version: '2.99'
|
|
23
|
-
|
|
22
|
+
name: logstash-core-plugin-api
|
|
23
|
+
prerelease: false
|
|
24
|
+
type: :runtime
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
24
26
|
requirements:
|
|
25
27
|
- - ">="
|
|
26
28
|
- !ruby/object:Gem::Version
|
|
@@ -28,11 +30,8 @@ dependencies:
|
|
|
28
30
|
- - "<="
|
|
29
31
|
- !ruby/object:Gem::Version
|
|
30
32
|
version: '2.99'
|
|
31
|
-
prerelease: false
|
|
32
|
-
type: :runtime
|
|
33
33
|
- !ruby/object:Gem::Dependency
|
|
34
|
-
|
|
35
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
35
|
requirements:
|
|
37
36
|
- - ">="
|
|
38
37
|
- !ruby/object:Gem::Version
|
|
@@ -40,7 +39,10 @@ dependencies:
|
|
|
40
39
|
- - "<"
|
|
41
40
|
- !ruby/object:Gem::Version
|
|
42
41
|
version: 6.0.0
|
|
43
|
-
|
|
42
|
+
name: elasticsearch
|
|
43
|
+
prerelease: false
|
|
44
|
+
type: :runtime
|
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
44
46
|
requirements:
|
|
45
47
|
- - ">="
|
|
46
48
|
- !ruby/object:Gem::Version
|
|
@@ -48,36 +50,34 @@ dependencies:
|
|
|
48
50
|
- - "<"
|
|
49
51
|
- !ruby/object:Gem::Version
|
|
50
52
|
version: 6.0.0
|
|
51
|
-
prerelease: false
|
|
52
|
-
type: :runtime
|
|
53
53
|
- !ruby/object:Gem::Dependency
|
|
54
|
-
name: logstash-codec-json
|
|
55
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
56
|
-
requirements:
|
|
57
|
-
- - ">="
|
|
58
|
-
- !ruby/object:Gem::Version
|
|
59
|
-
version: '0'
|
|
60
54
|
requirement: !ruby/object:Gem::Requirement
|
|
61
55
|
requirements:
|
|
62
56
|
- - ">="
|
|
63
57
|
- !ruby/object:Gem::Version
|
|
64
58
|
version: '0'
|
|
59
|
+
name: logstash-codec-json
|
|
65
60
|
prerelease: false
|
|
66
61
|
type: :runtime
|
|
67
|
-
- !ruby/object:Gem::Dependency
|
|
68
|
-
name: logstash-devutils
|
|
69
62
|
version_requirements: !ruby/object:Gem::Requirement
|
|
70
63
|
requirements:
|
|
71
64
|
- - ">="
|
|
72
65
|
- !ruby/object:Gem::Version
|
|
73
66
|
version: '0'
|
|
67
|
+
- !ruby/object:Gem::Dependency
|
|
74
68
|
requirement: !ruby/object:Gem::Requirement
|
|
75
69
|
requirements:
|
|
76
70
|
- - ">="
|
|
77
71
|
- !ruby/object:Gem::Version
|
|
78
72
|
version: '0'
|
|
73
|
+
name: logstash-devutils
|
|
79
74
|
prerelease: false
|
|
80
75
|
type: :development
|
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - ">="
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: '0'
|
|
81
81
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
|
82
82
|
email: info@elastic.co
|
|
83
83
|
executables: []
|
|
@@ -90,6 +90,7 @@ files:
|
|
|
90
90
|
- LICENSE
|
|
91
91
|
- NOTICE.TXT
|
|
92
92
|
- README.md
|
|
93
|
+
- docs/index.asciidoc
|
|
93
94
|
- lib/logstash/inputs/elasticsearch.rb
|
|
94
95
|
- logstash-input-elasticsearch.gemspec
|
|
95
96
|
- spec/inputs/elasticsearch_spec.rb
|