logstash-input-elasticsearch 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +3 -0
- data/lib/logstash/inputs/elasticsearch.rb +14 -13
- data/logstash-input-elasticsearch.gemspec +1 -1
- metadata +20 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4cfbfda39b8771c3e872329b18d475a9967f3205
|
4
|
+
data.tar.gz: 3001ba2198fea536baaf5c28398f5a248a2719c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b29568264d63c5049f7b61dce175906558c2d381332b92408d8ec1365c607b1df5d71ab7202e03c4a8ac051d6f4fbeb2f204e35f35fddce1fa43a89f9524fdc
|
7
|
+
data.tar.gz: 2e8696b2b59f0d460843320b6391ea92304220b6fe17c0325a81466f6a92bf295d3847996b7c77f0f781ba8aaa42920f70211f4fc2419d6b9185845a7fee2840
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- Refactored field references and cleanups
|
3
|
+
|
1
4
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
5
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
6
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
7
|
- Dependency on logstash-core update to 2.0
|
5
8
|
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-elasticsearch-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -60,7 +60,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
60
60
|
# It might be important to note, with regards to metadata, that if you're
|
61
61
|
# ingesting documents with the intent to re-index them (or just update them)
|
62
62
|
# that the `action` option in the elasticsearch output want's to know how to
|
63
|
-
# handle those things. It can be dynamically assigned with a field
|
63
|
+
# handle those things. It can be dynamically assigned with a field
|
64
64
|
# added to the metadata.
|
65
65
|
#
|
66
66
|
# Example
|
@@ -86,7 +86,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
86
86
|
config :docinfo, :validate => :boolean, :default => false
|
87
87
|
|
88
88
|
# Where to move the Elasticsearch document information by default we use the @metadata field.
|
89
|
-
config :docinfo_target, :validate=> :string, :default =>
|
89
|
+
config :docinfo_target, :validate=> :string, :default => LogStash::Event::METADATA
|
90
90
|
|
91
91
|
# List of document metadata to move to the `docinfo_target` field
|
92
92
|
# To learn more about Elasticsearch metadata fields read
|
@@ -105,7 +105,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
105
105
|
# SSL Certificate Authority file
|
106
106
|
config :ca_file, :validate => :path
|
107
107
|
|
108
|
-
public
|
109
108
|
def register
|
110
109
|
require "elasticsearch"
|
111
110
|
|
@@ -141,9 +140,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
141
140
|
@client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
|
142
141
|
end
|
143
142
|
|
144
|
-
public
|
145
143
|
def run(output_queue)
|
146
|
-
|
147
144
|
# get first wave of data
|
148
145
|
r = @client.search(@options)
|
149
146
|
|
@@ -160,38 +157,42 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
160
157
|
r = process_next_scroll(output_queue, r['_scroll_id'])
|
161
158
|
has_hits = r['has_hits']
|
162
159
|
end
|
163
|
-
end
|
160
|
+
end
|
164
161
|
|
165
162
|
private
|
163
|
+
|
166
164
|
def process_next_scroll(output_queue, scroll_id)
|
167
165
|
r = scroll_request(scroll_id)
|
168
166
|
r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
|
169
167
|
{'has_hits' => r['hits']['hits'].any?, '_scroll_id' => r['_scroll_id']}
|
170
168
|
end
|
171
169
|
|
172
|
-
private
|
173
170
|
def push_hit(hit, output_queue)
|
174
171
|
event = LogStash::Event.new(hit['_source'])
|
175
172
|
decorate(event)
|
176
173
|
|
177
174
|
if @docinfo
|
178
|
-
event[@docinfo_target]
|
175
|
+
# do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
|
176
|
+
docinfo_target = event[@docinfo_target] || {}
|
179
177
|
|
180
|
-
unless
|
181
|
-
@logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the
|
178
|
+
unless docinfo_target.is_a?(Hash)
|
179
|
+
@logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event)
|
182
180
|
|
181
|
+
# TODO: (colin) I am not sure raising is a good strategy here?
|
183
182
|
raise Exception.new("Elasticsearch input: incompatible event")
|
184
183
|
end
|
185
184
|
|
186
185
|
@docinfo_fields.each do |field|
|
187
|
-
|
186
|
+
docinfo_target[field] = hit[field]
|
188
187
|
end
|
188
|
+
|
189
|
+
event[@docinfo_target] = docinfo_target
|
189
190
|
end
|
191
|
+
|
190
192
|
output_queue << event
|
191
193
|
end
|
192
194
|
|
193
|
-
private
|
194
195
|
def scroll_request scroll_id
|
195
196
|
@client.scroll(:body => scroll_id, :scroll => @scroll)
|
196
197
|
end
|
197
|
-
end
|
198
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Read from an Elasticsearch cluster, based on search query results"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,8 +28,11 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
|
34
|
+
name: elasticsearch
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
35
36
|
requirements:
|
36
37
|
- - '>='
|
37
38
|
- !ruby/object:Gem::Version
|
@@ -39,10 +40,7 @@ dependencies:
|
|
39
40
|
- - ~>
|
40
41
|
- !ruby/object:Gem::Version
|
41
42
|
version: '1.0'
|
42
|
-
|
43
|
-
prerelease: false
|
44
|
-
type: :runtime
|
45
|
-
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
46
44
|
requirements:
|
47
45
|
- - '>='
|
48
46
|
- !ruby/object:Gem::Version
|
@@ -50,34 +48,36 @@ dependencies:
|
|
50
48
|
- - ~>
|
51
49
|
- !ruby/object:Gem::Version
|
52
50
|
version: '1.0'
|
51
|
+
prerelease: false
|
52
|
+
type: :runtime
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
+
name: logstash-codec-json
|
55
|
+
version_requirements: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - '>='
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '0'
|
54
60
|
requirement: !ruby/object:Gem::Requirement
|
55
61
|
requirements:
|
56
62
|
- - '>='
|
57
63
|
- !ruby/object:Gem::Version
|
58
64
|
version: '0'
|
59
|
-
name: logstash-codec-json
|
60
65
|
prerelease: false
|
61
66
|
type: :runtime
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
name: logstash-devutils
|
62
69
|
version_requirements: !ruby/object:Gem::Requirement
|
63
70
|
requirements:
|
64
71
|
- - '>='
|
65
72
|
- !ruby/object:Gem::Version
|
66
73
|
version: '0'
|
67
|
-
- !ruby/object:Gem::Dependency
|
68
74
|
requirement: !ruby/object:Gem::Requirement
|
69
75
|
requirements:
|
70
76
|
- - '>='
|
71
77
|
- !ruby/object:Gem::Version
|
72
78
|
version: '0'
|
73
|
-
name: logstash-devutils
|
74
79
|
prerelease: false
|
75
80
|
type: :development
|
76
|
-
version_requirements: !ruby/object:Gem::Requirement
|
77
|
-
requirements:
|
78
|
-
- - '>='
|
79
|
-
- !ruby/object:Gem::Version
|
80
|
-
version: '0'
|
81
81
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
82
82
|
email: info@elastic.co
|
83
83
|
executables: []
|