logstash-output-elasticsearch 2.6.2-java → 2.7.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +2 -2
- data/lib/logstash/outputs/elasticsearch/common.rb +9 -1
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +3 -0
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/pipeline_spec.rb +75 -0
- data/spec/integration/outputs/templates_spec.rb +1 -1
- metadata +47 -45
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 74f2ac96bc6550963861463f730f02b3f7db5b6e
|
4
|
+
data.tar.gz: 05c1c72ec132dac8ef7284b08a2180d9e72e2bb4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f38466236f48b6162e57e7cbe5b1eebe175847a53dd98690291f74a7db328f75cff86d2abe8a22707cb662c2c7f3db04d53a38a09042a278b7e263828812cdc0
|
7
|
+
data.tar.gz: 0bf1bdf857e923e3103045fa7e1bb9f5df585a3c775db9ef632c2fa9a88eeefbff357e12ee89a024d6b7247191a803dddadfaee846382ddbedfdbfac8b49f7ee
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -68,7 +68,7 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
|
68
68
|
- Install plugin
|
69
69
|
```sh
|
70
70
|
# Logstash 2.3 and higher
|
71
|
-
bin/
|
71
|
+
bin/logstash-plugin install --no-verify
|
72
72
|
|
73
73
|
# Prior to Logstash 2.3
|
74
74
|
bin/plugin install --no-verify
|
@@ -91,7 +91,7 @@ gem build logstash-filter-awesome.gemspec
|
|
91
91
|
- Install the plugin from the Logstash home
|
92
92
|
```sh
|
93
93
|
# Logstash 2.3 and higher
|
94
|
-
bin/
|
94
|
+
bin/logstash-plugin install --no-verify
|
95
95
|
|
96
96
|
# Prior to Logstash 2.3
|
97
97
|
bin/plugin install --no-verify
|
@@ -134,12 +134,20 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
134
134
|
:_routing => @routing ? event.sprintf(@routing) : nil
|
135
135
|
}
|
136
136
|
|
137
|
-
|
137
|
+
if @pipeline
|
138
|
+
params[:pipeline] = @pipeline
|
139
|
+
end
|
140
|
+
|
141
|
+
if @parent
|
142
|
+
params[:parent] = event.sprintf(@parent)
|
143
|
+
end
|
144
|
+
|
138
145
|
if @action == 'update'
|
139
146
|
params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
|
140
147
|
params[:_script] = event.sprintf(@script) if @script != ""
|
141
148
|
params[:_retry_on_conflict] = @retry_on_conflict
|
142
149
|
end
|
150
|
+
|
143
151
|
params
|
144
152
|
end
|
145
153
|
|
@@ -156,6 +156,9 @@ module LogStash; module Outputs; class ElasticSearch
|
|
156
156
|
# See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
|
157
157
|
# for more info
|
158
158
|
mod.config :retry_on_conflict, :validate => :number, :default => 1
|
159
|
+
|
160
|
+
# Set which ingest pipeline you wish to execute for an event
|
161
|
+
mod.config :pipeline, :validate => :string, :default => nil
|
159
162
|
end
|
160
163
|
end
|
161
164
|
end end end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-elasticsearch'
|
4
|
-
s.version = '2.
|
4
|
+
s.version = '2.7.0'
|
5
5
|
s.licenses = ['apache-2.0']
|
6
6
|
s.summary = "Logstash Output to Elasticsearch"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require_relative "../../../spec/es_spec_helper"
|
2
|
+
|
3
|
+
describe "Ingest pipeline execution behavior", :integration => true, :version_5x => true do
|
4
|
+
subject! do
|
5
|
+
require "logstash/outputs/elasticsearch"
|
6
|
+
settings = {
|
7
|
+
"hosts" => "#{get_host_port()}",
|
8
|
+
"pipeline" => "apache-logs"
|
9
|
+
}
|
10
|
+
next LogStash::Outputs::ElasticSearch.new(settings)
|
11
|
+
end
|
12
|
+
|
13
|
+
let(:ftw_client) { FTW::Agent.new }
|
14
|
+
let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
|
15
|
+
let(:apache_logs_pipeline) { '
|
16
|
+
{
|
17
|
+
"description" : "Pipeline to parse Apache logs",
|
18
|
+
"processors" : [
|
19
|
+
{
|
20
|
+
"grok": {
|
21
|
+
"field": "message",
|
22
|
+
"pattern": "%{COMBINEDAPACHELOG}"
|
23
|
+
}
|
24
|
+
}
|
25
|
+
]
|
26
|
+
}'
|
27
|
+
}
|
28
|
+
|
29
|
+
before :each do
|
30
|
+
# Delete all templates first.
|
31
|
+
require "elasticsearch"
|
32
|
+
|
33
|
+
# Clean ES of data before we start.
|
34
|
+
@es = get_client
|
35
|
+
@es.indices.delete_template(:name => "*")
|
36
|
+
|
37
|
+
# This can fail if there are no indexes, ignore failure.
|
38
|
+
@es.indices.delete(:index => "*") rescue nil
|
39
|
+
|
40
|
+
# delete existing ingest pipeline
|
41
|
+
req = ftw_client.delete(ingest_url)
|
42
|
+
ftw_client.execute(req)
|
43
|
+
|
44
|
+
# register pipeline
|
45
|
+
req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
|
46
|
+
ftw_client.execute(req)
|
47
|
+
|
48
|
+
#TODO: Use esclient
|
49
|
+
#@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
|
50
|
+
|
51
|
+
subject.register
|
52
|
+
subject.receive(LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'))
|
53
|
+
subject.flush
|
54
|
+
@es.indices.refresh
|
55
|
+
|
56
|
+
#Wait or fail until everything's indexed.
|
57
|
+
Stud::try(20.times) do
|
58
|
+
r = @es.search
|
59
|
+
insist { r["hits"]["total"] } == 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
it "indexes using the proper pipeline" do
|
64
|
+
results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
|
65
|
+
insist { results["hits"]["total"] } == 1
|
66
|
+
insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
|
67
|
+
insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
|
68
|
+
insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
|
69
|
+
insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
|
70
|
+
insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
|
71
|
+
insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
|
72
|
+
insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
|
73
|
+
insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
|
74
|
+
end
|
75
|
+
end
|
@@ -73,7 +73,7 @@ describe "index template expected behavior", :integration => true do
|
|
73
73
|
end
|
74
74
|
|
75
75
|
it "make [geoip][location] a geo_point" do
|
76
|
-
results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
|
76
|
+
results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
|
77
77
|
insist { results["hits"]["total"] } == 1
|
78
78
|
insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
|
79
79
|
end
|
metadata
CHANGED
@@ -1,39 +1,30 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.7.0
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-04-
|
11
|
+
date: 2016-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: concurrent-ruby
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - '>='
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '0'
|
19
|
+
name: concurrent-ruby
|
25
20
|
prerelease: false
|
26
21
|
type: :runtime
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: elasticsearch
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
24
|
- - '>='
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
34
|
-
|
35
|
-
- !ruby/object:Gem::Version
|
36
|
-
version: '1.0'
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
37
28
|
requirement: !ruby/object:Gem::Requirement
|
38
29
|
requirements:
|
39
30
|
- - '>='
|
@@ -42,18 +33,18 @@ dependencies:
|
|
42
33
|
- - ~>
|
43
34
|
- !ruby/object:Gem::Version
|
44
35
|
version: '1.0'
|
36
|
+
name: elasticsearch
|
45
37
|
prerelease: false
|
46
38
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: stud
|
49
39
|
version_requirements: !ruby/object:Gem::Requirement
|
50
40
|
requirements:
|
51
41
|
- - '>='
|
52
42
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
43
|
+
version: 1.0.13
|
54
44
|
- - ~>
|
55
45
|
- !ruby/object:Gem::Version
|
56
|
-
version: '
|
46
|
+
version: '1.0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
57
48
|
requirement: !ruby/object:Gem::Requirement
|
58
49
|
requirements:
|
59
50
|
- - '>='
|
@@ -62,74 +53,74 @@ dependencies:
|
|
62
53
|
- - ~>
|
63
54
|
- !ruby/object:Gem::Version
|
64
55
|
version: '0.0'
|
56
|
+
name: stud
|
65
57
|
prerelease: false
|
66
58
|
type: :runtime
|
67
|
-
- !ruby/object:Gem::Dependency
|
68
|
-
name: cabin
|
69
59
|
version_requirements: !ruby/object:Gem::Requirement
|
70
60
|
requirements:
|
61
|
+
- - '>='
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: 0.0.17
|
71
64
|
- - ~>
|
72
65
|
- !ruby/object:Gem::Version
|
73
|
-
version: '0.
|
66
|
+
version: '0.0'
|
67
|
+
- !ruby/object:Gem::Dependency
|
74
68
|
requirement: !ruby/object:Gem::Requirement
|
75
69
|
requirements:
|
76
70
|
- - ~>
|
77
71
|
- !ruby/object:Gem::Version
|
78
72
|
version: '0.6'
|
73
|
+
name: cabin
|
79
74
|
prerelease: false
|
80
75
|
type: :runtime
|
81
|
-
- !ruby/object:Gem::Dependency
|
82
|
-
name: logstash-core-plugin-api
|
83
76
|
version_requirements: !ruby/object:Gem::Requirement
|
84
77
|
requirements:
|
85
78
|
- - ~>
|
86
79
|
- !ruby/object:Gem::Version
|
87
|
-
version: '
|
80
|
+
version: '0.6'
|
81
|
+
- !ruby/object:Gem::Dependency
|
88
82
|
requirement: !ruby/object:Gem::Requirement
|
89
83
|
requirements:
|
90
84
|
- - ~>
|
91
85
|
- !ruby/object:Gem::Version
|
92
86
|
version: '1.0'
|
87
|
+
name: logstash-core-plugin-api
|
93
88
|
prerelease: false
|
94
89
|
type: :runtime
|
95
|
-
- !ruby/object:Gem::Dependency
|
96
|
-
name: ftw
|
97
90
|
version_requirements: !ruby/object:Gem::Requirement
|
98
91
|
requirements:
|
99
92
|
- - ~>
|
100
93
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
94
|
+
version: '1.0'
|
95
|
+
- !ruby/object:Gem::Dependency
|
102
96
|
requirement: !ruby/object:Gem::Requirement
|
103
97
|
requirements:
|
104
98
|
- - ~>
|
105
99
|
- !ruby/object:Gem::Version
|
106
100
|
version: 0.0.42
|
101
|
+
name: ftw
|
107
102
|
prerelease: false
|
108
103
|
type: :development
|
109
|
-
- !ruby/object:Gem::Dependency
|
110
|
-
name: logstash-codec-plain
|
111
104
|
version_requirements: !ruby/object:Gem::Requirement
|
112
105
|
requirements:
|
113
|
-
- -
|
106
|
+
- - ~>
|
114
107
|
- !ruby/object:Gem::Version
|
115
|
-
version:
|
108
|
+
version: 0.0.42
|
109
|
+
- !ruby/object:Gem::Dependency
|
116
110
|
requirement: !ruby/object:Gem::Requirement
|
117
111
|
requirements:
|
118
112
|
- - '>='
|
119
113
|
- !ruby/object:Gem::Version
|
120
114
|
version: '0'
|
115
|
+
name: logstash-codec-plain
|
121
116
|
prerelease: false
|
122
117
|
type: :development
|
123
|
-
- !ruby/object:Gem::Dependency
|
124
|
-
name: manticore
|
125
118
|
version_requirements: !ruby/object:Gem::Requirement
|
126
119
|
requirements:
|
127
120
|
- - '>='
|
128
121
|
- !ruby/object:Gem::Version
|
129
|
-
version: 0
|
130
|
-
|
131
|
-
- !ruby/object:Gem::Version
|
132
|
-
version: 1.0.0
|
122
|
+
version: '0'
|
123
|
+
- !ruby/object:Gem::Dependency
|
133
124
|
requirement: !ruby/object:Gem::Requirement
|
134
125
|
requirements:
|
135
126
|
- - '>='
|
@@ -138,50 +129,59 @@ dependencies:
|
|
138
129
|
- - <
|
139
130
|
- !ruby/object:Gem::Version
|
140
131
|
version: 1.0.0
|
132
|
+
name: manticore
|
141
133
|
prerelease: false
|
142
134
|
type: :runtime
|
143
|
-
- !ruby/object:Gem::Dependency
|
144
|
-
name: logstash-devutils
|
145
135
|
version_requirements: !ruby/object:Gem::Requirement
|
146
136
|
requirements:
|
147
137
|
- - '>='
|
148
138
|
- !ruby/object:Gem::Version
|
149
|
-
version:
|
139
|
+
version: 0.5.4
|
140
|
+
- - <
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: 1.0.0
|
143
|
+
- !ruby/object:Gem::Dependency
|
150
144
|
requirement: !ruby/object:Gem::Requirement
|
151
145
|
requirements:
|
152
146
|
- - '>='
|
153
147
|
- !ruby/object:Gem::Version
|
154
148
|
version: '0'
|
149
|
+
name: logstash-devutils
|
155
150
|
prerelease: false
|
156
151
|
type: :development
|
157
|
-
- !ruby/object:Gem::Dependency
|
158
|
-
name: longshoreman
|
159
152
|
version_requirements: !ruby/object:Gem::Requirement
|
160
153
|
requirements:
|
161
154
|
- - '>='
|
162
155
|
- !ruby/object:Gem::Version
|
163
156
|
version: '0'
|
157
|
+
- !ruby/object:Gem::Dependency
|
164
158
|
requirement: !ruby/object:Gem::Requirement
|
165
159
|
requirements:
|
166
160
|
- - '>='
|
167
161
|
- !ruby/object:Gem::Version
|
168
162
|
version: '0'
|
163
|
+
name: longshoreman
|
169
164
|
prerelease: false
|
170
165
|
type: :development
|
171
|
-
- !ruby/object:Gem::Dependency
|
172
|
-
name: flores
|
173
166
|
version_requirements: !ruby/object:Gem::Requirement
|
174
167
|
requirements:
|
175
168
|
- - '>='
|
176
169
|
- !ruby/object:Gem::Version
|
177
170
|
version: '0'
|
171
|
+
- !ruby/object:Gem::Dependency
|
178
172
|
requirement: !ruby/object:Gem::Requirement
|
179
173
|
requirements:
|
180
174
|
- - '>='
|
181
175
|
- !ruby/object:Gem::Version
|
182
176
|
version: '0'
|
177
|
+
name: flores
|
183
178
|
prerelease: false
|
184
179
|
type: :development
|
180
|
+
version_requirements: !ruby/object:Gem::Requirement
|
181
|
+
requirements:
|
182
|
+
- - '>='
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
version: '0'
|
185
185
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
186
186
|
email: info@elastic.co
|
187
187
|
executables: []
|
@@ -210,6 +210,7 @@ files:
|
|
210
210
|
- spec/integration/outputs/create_spec.rb
|
211
211
|
- spec/integration/outputs/index_spec.rb
|
212
212
|
- spec/integration/outputs/parent_spec.rb
|
213
|
+
- spec/integration/outputs/pipeline_spec.rb
|
213
214
|
- spec/integration/outputs/retry_spec.rb
|
214
215
|
- spec/integration/outputs/routing_spec.rb
|
215
216
|
- spec/integration/outputs/secure_spec.rb
|
@@ -243,7 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
243
244
|
version: '0'
|
244
245
|
requirements: []
|
245
246
|
rubyforge_project:
|
246
|
-
rubygems_version: 2.4.
|
247
|
+
rubygems_version: 2.4.5
|
247
248
|
signing_key:
|
248
249
|
specification_version: 4
|
249
250
|
summary: Logstash Output to Elasticsearch
|
@@ -255,6 +256,7 @@ test_files:
|
|
255
256
|
- spec/integration/outputs/create_spec.rb
|
256
257
|
- spec/integration/outputs/index_spec.rb
|
257
258
|
- spec/integration/outputs/parent_spec.rb
|
259
|
+
- spec/integration/outputs/pipeline_spec.rb
|
258
260
|
- spec/integration/outputs/retry_spec.rb
|
259
261
|
- spec/integration/outputs/routing_spec.rb
|
260
262
|
- spec/integration/outputs/secure_spec.rb
|