logstash-output-elasticsearch 2.6.2-java → 2.7.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/README.md +2 -2
  4. data/lib/logstash/outputs/elasticsearch/common.rb +9 -1
  5. data/lib/logstash/outputs/elasticsearch/common_configs.rb +3 -0
  6. data/logstash-output-elasticsearch.gemspec +1 -1
  7. data/spec/integration/outputs/pipeline_spec.rb +75 -0
  8. data/spec/integration/outputs/templates_spec.rb +1 -1
  9. metadata +47 -45
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0e498531e88fe7f64902a58f2b8c78692e53fe50
4
- data.tar.gz: 649251ac66818877c41c3fd09415fcabcbaf4887
3
+ metadata.gz: 74f2ac96bc6550963861463f730f02b3f7db5b6e
4
+ data.tar.gz: 05c1c72ec132dac8ef7284b08a2180d9e72e2bb4
5
5
  SHA512:
6
- metadata.gz: 37f1ad94ed0a96818dd4abe9290d5b40b43a61956fda40e2f68552579b0a6f7764b3ed1f3cef3fa6e228a0bbe63960a1f37a8d6fd172d10320d52d053f6fe1f2
7
- data.tar.gz: 65d29533998b3a822c3a98a9867fd987158c89180060e4685797bcb94891356669827354fe86a245dbe736d9f516b7c2664c1f01cd97ef465f1199ac3fe82078
6
+ metadata.gz: f38466236f48b6162e57e7cbe5b1eebe175847a53dd98690291f74a7db328f75cff86d2abe8a22707cb662c2c7f3db04d53a38a09042a278b7e263828812cdc0
7
+ data.tar.gz: 0bf1bdf857e923e3103045fa7e1bb9f5df585a3c775db9ef632c2fa9a88eeefbff357e12ee89a024d6b7247191a803dddadfaee846382ddbedfdbfac8b49f7ee
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 2.7.0
2
+ - Add `pipeline` configuration option for setting an ingest pipeline to run upon indexing
3
+
1
4
  ## 2.6.2
2
5
  - Fix bug where update index actions would not work with events with 'data' field
3
6
 
data/README.md CHANGED
@@ -68,7 +68,7 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
68
68
  - Install plugin
69
69
  ```sh
70
70
  # Logstash 2.3 and higher
71
- bin/logstah-plugin install --no-verify
71
+ bin/logstash-plugin install --no-verify
72
72
 
73
73
  # Prior to Logstash 2.3
74
74
  bin/plugin install --no-verify
@@ -91,7 +91,7 @@ gem build logstash-filter-awesome.gemspec
91
91
  - Install the plugin from the Logstash home
92
92
  ```sh
93
93
  # Logstash 2.3 and higher
94
- bin/logstah-plugin install --no-verify
94
+ bin/logstash-plugin install --no-verify
95
95
 
96
96
  # Prior to Logstash 2.3
97
97
  bin/plugin install --no-verify
data/lib/logstash/outputs/elasticsearch/common.rb CHANGED
@@ -134,12 +134,20 @@ module LogStash; module Outputs; class ElasticSearch;
134
134
  :_routing => @routing ? event.sprintf(@routing) : nil
135
135
  }
136
136
 
137
- params[:parent] = event.sprintf(@parent) if @parent
137
+ if @pipeline
138
+ params[:pipeline] = @pipeline
139
+ end
140
+
141
+ if @parent
142
+ params[:parent] = event.sprintf(@parent)
143
+ end
144
+
138
145
  if @action == 'update'
139
146
  params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
140
147
  params[:_script] = event.sprintf(@script) if @script != ""
141
148
  params[:_retry_on_conflict] = @retry_on_conflict
142
149
  end
150
+
143
151
  params
144
152
  end
145
153
 
data/lib/logstash/outputs/elasticsearch/common_configs.rb CHANGED
@@ -156,6 +156,9 @@ module LogStash; module Outputs; class ElasticSearch
156
156
  # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
157
157
  # for more info
158
158
  mod.config :retry_on_conflict, :validate => :number, :default => 1
159
+
160
+ # Set which ingest pipeline you wish to execute for an event
161
+ mod.config :pipeline, :validate => :string, :default => nil
159
162
  end
160
163
  end
161
164
  end end end
data/logstash-output-elasticsearch.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '2.6.2'
4
+ s.version = '2.7.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
data/spec/integration/outputs/pipeline_spec.rb ADDED
@@ -0,0 +1,75 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ describe "Ingest pipeline execution behavior", :integration => true, :version_5x => true do
4
+ subject! do
5
+ require "logstash/outputs/elasticsearch"
6
+ settings = {
7
+ "hosts" => "#{get_host_port()}",
8
+ "pipeline" => "apache-logs"
9
+ }
10
+ next LogStash::Outputs::ElasticSearch.new(settings)
11
+ end
12
+
13
+ let(:ftw_client) { FTW::Agent.new }
14
+ let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
15
+ let(:apache_logs_pipeline) { '
16
+ {
17
+ "description" : "Pipeline to parse Apache logs",
18
+ "processors" : [
19
+ {
20
+ "grok": {
21
+ "field": "message",
22
+ "pattern": "%{COMBINEDAPACHELOG}"
23
+ }
24
+ }
25
+ ]
26
+ }'
27
+ }
28
+
29
+ before :each do
30
+ # Delete all templates first.
31
+ require "elasticsearch"
32
+
33
+ # Clean ES of data before we start.
34
+ @es = get_client
35
+ @es.indices.delete_template(:name => "*")
36
+
37
+ # This can fail if there are no indexes, ignore failure.
38
+ @es.indices.delete(:index => "*") rescue nil
39
+
40
+ # delete existing ingest pipeline
41
+ req = ftw_client.delete(ingest_url)
42
+ ftw_client.execute(req)
43
+
44
+ # register pipeline
45
+ req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
46
+ ftw_client.execute(req)
47
+
48
+ #TODO: Use esclient
49
+ #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
50
+
51
+ subject.register
52
+ subject.receive(LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'))
53
+ subject.flush
54
+ @es.indices.refresh
55
+
56
+ #Wait or fail until everything's indexed.
57
+ Stud::try(20.times) do
58
+ r = @es.search
59
+ insist { r["hits"]["total"] } == 1
60
+ end
61
+ end
62
+
63
+ it "indexes using the proper pipeline" do
64
+ results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
65
+ insist { results["hits"]["total"] } == 1
66
+ insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
67
+ insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
68
+ insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
69
+ insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
70
+ insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
71
+ insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
72
+ insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
73
+ insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
74
+ end
75
+ end
data/spec/integration/outputs/templates_spec.rb CHANGED
@@ -73,7 +73,7 @@ describe "index template expected behavior", :integration => true do
73
73
  end
74
74
 
75
75
  it "make [geoip][location] a geo_point" do
76
- results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
76
+ results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
77
77
  insist { results["hits"]["total"] } == 1
78
78
  insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
79
79
  end
metadata CHANGED
@@ -1,39 +1,30 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.6.2
4
+ version: 2.7.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-20 00:00:00.000000000 Z
11
+ date: 2016-04-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: concurrent-ruby
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '>='
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - '>='
23
17
  - !ruby/object:Gem::Version
24
18
  version: '0'
19
+ name: concurrent-ruby
25
20
  prerelease: false
26
21
  type: :runtime
27
- - !ruby/object:Gem::Dependency
28
- name: elasticsearch
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
24
  - - '>='
32
25
  - !ruby/object:Gem::Version
33
- version: 1.0.13
34
- - - ~>
35
- - !ruby/object:Gem::Version
36
- version: '1.0'
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
37
28
  requirement: !ruby/object:Gem::Requirement
38
29
  requirements:
39
30
  - - '>='
@@ -42,18 +33,18 @@ dependencies:
42
33
  - - ~>
43
34
  - !ruby/object:Gem::Version
44
35
  version: '1.0'
36
+ name: elasticsearch
45
37
  prerelease: false
46
38
  type: :runtime
47
- - !ruby/object:Gem::Dependency
48
- name: stud
49
39
  version_requirements: !ruby/object:Gem::Requirement
50
40
  requirements:
51
41
  - - '>='
52
42
  - !ruby/object:Gem::Version
53
- version: 0.0.17
43
+ version: 1.0.13
54
44
  - - ~>
55
45
  - !ruby/object:Gem::Version
56
- version: '0.0'
46
+ version: '1.0'
47
+ - !ruby/object:Gem::Dependency
57
48
  requirement: !ruby/object:Gem::Requirement
58
49
  requirements:
59
50
  - - '>='
@@ -62,74 +53,74 @@ dependencies:
62
53
  - - ~>
63
54
  - !ruby/object:Gem::Version
64
55
  version: '0.0'
56
+ name: stud
65
57
  prerelease: false
66
58
  type: :runtime
67
- - !ruby/object:Gem::Dependency
68
- name: cabin
69
59
  version_requirements: !ruby/object:Gem::Requirement
70
60
  requirements:
61
+ - - '>='
62
+ - !ruby/object:Gem::Version
63
+ version: 0.0.17
71
64
  - - ~>
72
65
  - !ruby/object:Gem::Version
73
- version: '0.6'
66
+ version: '0.0'
67
+ - !ruby/object:Gem::Dependency
74
68
  requirement: !ruby/object:Gem::Requirement
75
69
  requirements:
76
70
  - - ~>
77
71
  - !ruby/object:Gem::Version
78
72
  version: '0.6'
73
+ name: cabin
79
74
  prerelease: false
80
75
  type: :runtime
81
- - !ruby/object:Gem::Dependency
82
- name: logstash-core-plugin-api
83
76
  version_requirements: !ruby/object:Gem::Requirement
84
77
  requirements:
85
78
  - - ~>
86
79
  - !ruby/object:Gem::Version
87
- version: '1.0'
80
+ version: '0.6'
81
+ - !ruby/object:Gem::Dependency
88
82
  requirement: !ruby/object:Gem::Requirement
89
83
  requirements:
90
84
  - - ~>
91
85
  - !ruby/object:Gem::Version
92
86
  version: '1.0'
87
+ name: logstash-core-plugin-api
93
88
  prerelease: false
94
89
  type: :runtime
95
- - !ruby/object:Gem::Dependency
96
- name: ftw
97
90
  version_requirements: !ruby/object:Gem::Requirement
98
91
  requirements:
99
92
  - - ~>
100
93
  - !ruby/object:Gem::Version
101
- version: 0.0.42
94
+ version: '1.0'
95
+ - !ruby/object:Gem::Dependency
102
96
  requirement: !ruby/object:Gem::Requirement
103
97
  requirements:
104
98
  - - ~>
105
99
  - !ruby/object:Gem::Version
106
100
  version: 0.0.42
101
+ name: ftw
107
102
  prerelease: false
108
103
  type: :development
109
- - !ruby/object:Gem::Dependency
110
- name: logstash-codec-plain
111
104
  version_requirements: !ruby/object:Gem::Requirement
112
105
  requirements:
113
- - - '>='
106
+ - - ~>
114
107
  - !ruby/object:Gem::Version
115
- version: '0'
108
+ version: 0.0.42
109
+ - !ruby/object:Gem::Dependency
116
110
  requirement: !ruby/object:Gem::Requirement
117
111
  requirements:
118
112
  - - '>='
119
113
  - !ruby/object:Gem::Version
120
114
  version: '0'
115
+ name: logstash-codec-plain
121
116
  prerelease: false
122
117
  type: :development
123
- - !ruby/object:Gem::Dependency
124
- name: manticore
125
118
  version_requirements: !ruby/object:Gem::Requirement
126
119
  requirements:
127
120
  - - '>='
128
121
  - !ruby/object:Gem::Version
129
- version: 0.5.4
130
- - - <
131
- - !ruby/object:Gem::Version
132
- version: 1.0.0
122
+ version: '0'
123
+ - !ruby/object:Gem::Dependency
133
124
  requirement: !ruby/object:Gem::Requirement
134
125
  requirements:
135
126
  - - '>='
@@ -138,50 +129,59 @@ dependencies:
138
129
  - - <
139
130
  - !ruby/object:Gem::Version
140
131
  version: 1.0.0
132
+ name: manticore
141
133
  prerelease: false
142
134
  type: :runtime
143
- - !ruby/object:Gem::Dependency
144
- name: logstash-devutils
145
135
  version_requirements: !ruby/object:Gem::Requirement
146
136
  requirements:
147
137
  - - '>='
148
138
  - !ruby/object:Gem::Version
149
- version: '0'
139
+ version: 0.5.4
140
+ - - <
141
+ - !ruby/object:Gem::Version
142
+ version: 1.0.0
143
+ - !ruby/object:Gem::Dependency
150
144
  requirement: !ruby/object:Gem::Requirement
151
145
  requirements:
152
146
  - - '>='
153
147
  - !ruby/object:Gem::Version
154
148
  version: '0'
149
+ name: logstash-devutils
155
150
  prerelease: false
156
151
  type: :development
157
- - !ruby/object:Gem::Dependency
158
- name: longshoreman
159
152
  version_requirements: !ruby/object:Gem::Requirement
160
153
  requirements:
161
154
  - - '>='
162
155
  - !ruby/object:Gem::Version
163
156
  version: '0'
157
+ - !ruby/object:Gem::Dependency
164
158
  requirement: !ruby/object:Gem::Requirement
165
159
  requirements:
166
160
  - - '>='
167
161
  - !ruby/object:Gem::Version
168
162
  version: '0'
163
+ name: longshoreman
169
164
  prerelease: false
170
165
  type: :development
171
- - !ruby/object:Gem::Dependency
172
- name: flores
173
166
  version_requirements: !ruby/object:Gem::Requirement
174
167
  requirements:
175
168
  - - '>='
176
169
  - !ruby/object:Gem::Version
177
170
  version: '0'
171
+ - !ruby/object:Gem::Dependency
178
172
  requirement: !ruby/object:Gem::Requirement
179
173
  requirements:
180
174
  - - '>='
181
175
  - !ruby/object:Gem::Version
182
176
  version: '0'
177
+ name: flores
183
178
  prerelease: false
184
179
  type: :development
180
+ version_requirements: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - '>='
183
+ - !ruby/object:Gem::Version
184
+ version: '0'
185
185
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
186
186
  email: info@elastic.co
187
187
  executables: []
@@ -210,6 +210,7 @@ files:
210
210
  - spec/integration/outputs/create_spec.rb
211
211
  - spec/integration/outputs/index_spec.rb
212
212
  - spec/integration/outputs/parent_spec.rb
213
+ - spec/integration/outputs/pipeline_spec.rb
213
214
  - spec/integration/outputs/retry_spec.rb
214
215
  - spec/integration/outputs/routing_spec.rb
215
216
  - spec/integration/outputs/secure_spec.rb
@@ -243,7 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
243
244
  version: '0'
244
245
  requirements: []
245
246
  rubyforge_project:
246
- rubygems_version: 2.4.8
247
+ rubygems_version: 2.4.5
247
248
  signing_key:
248
249
  specification_version: 4
249
250
  summary: Logstash Output to Elasticsearch
@@ -255,6 +256,7 @@ test_files:
255
256
  - spec/integration/outputs/create_spec.rb
256
257
  - spec/integration/outputs/index_spec.rb
257
258
  - spec/integration/outputs/parent_spec.rb
259
+ - spec/integration/outputs/pipeline_spec.rb
258
260
  - spec/integration/outputs/retry_spec.rb
259
261
  - spec/integration/outputs/routing_spec.rb
260
262
  - spec/integration/outputs/secure_spec.rb