logstash-output-elasticsearch 2.6.2-java → 2.7.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/README.md +2 -2
  4. data/lib/logstash/outputs/elasticsearch/common.rb +9 -1
  5. data/lib/logstash/outputs/elasticsearch/common_configs.rb +3 -0
  6. data/logstash-output-elasticsearch.gemspec +1 -1
  7. data/spec/integration/outputs/pipeline_spec.rb +75 -0
  8. data/spec/integration/outputs/templates_spec.rb +1 -1
  9. metadata +47 -45
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0e498531e88fe7f64902a58f2b8c78692e53fe50
4
- data.tar.gz: 649251ac66818877c41c3fd09415fcabcbaf4887
3
+ metadata.gz: 74f2ac96bc6550963861463f730f02b3f7db5b6e
4
+ data.tar.gz: 05c1c72ec132dac8ef7284b08a2180d9e72e2bb4
5
5
  SHA512:
6
- metadata.gz: 37f1ad94ed0a96818dd4abe9290d5b40b43a61956fda40e2f68552579b0a6f7764b3ed1f3cef3fa6e228a0bbe63960a1f37a8d6fd172d10320d52d053f6fe1f2
7
- data.tar.gz: 65d29533998b3a822c3a98a9867fd987158c89180060e4685797bcb94891356669827354fe86a245dbe736d9f516b7c2664c1f01cd97ef465f1199ac3fe82078
6
+ metadata.gz: f38466236f48b6162e57e7cbe5b1eebe175847a53dd98690291f74a7db328f75cff86d2abe8a22707cb662c2c7f3db04d53a38a09042a278b7e263828812cdc0
7
+ data.tar.gz: 0bf1bdf857e923e3103045fa7e1bb9f5df585a3c775db9ef632c2fa9a88eeefbff357e12ee89a024d6b7247191a803dddadfaee846382ddbedfdbfac8b49f7ee
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 2.7.0
2
+ - Add `pipeline` configuration option for setting an ingest pipeline to run upon indexing
3
+
1
4
  ## 2.6.2
2
5
  - Fix bug where update index actions would not work with events with 'data' field
3
6
 
data/README.md CHANGED
@@ -68,7 +68,7 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
68
68
  - Install plugin
69
69
  ```sh
70
70
  # Logstash 2.3 and higher
71
- bin/logstah-plugin install --no-verify
71
+ bin/logstash-plugin install --no-verify
72
72
 
73
73
  # Prior to Logstash 2.3
74
74
  bin/plugin install --no-verify
@@ -91,7 +91,7 @@ gem build logstash-filter-awesome.gemspec
91
91
  - Install the plugin from the Logstash home
92
92
  ```sh
93
93
  # Logstash 2.3 and higher
94
- bin/logstah-plugin install --no-verify
94
+ bin/logstash-plugin install --no-verify
95
95
 
96
96
  # Prior to Logstash 2.3
97
97
  bin/plugin install --no-verify
data/lib/logstash/outputs/elasticsearch/common.rb CHANGED
@@ -134,12 +134,20 @@ module LogStash; module Outputs; class ElasticSearch;
134
134
  :_routing => @routing ? event.sprintf(@routing) : nil
135
135
  }
136
136
 
137
- params[:parent] = event.sprintf(@parent) if @parent
137
+ if @pipeline
138
+ params[:pipeline] = @pipeline
139
+ end
140
+
141
+ if @parent
142
+ params[:parent] = event.sprintf(@parent)
143
+ end
144
+
138
145
  if @action == 'update'
139
146
  params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
140
147
  params[:_script] = event.sprintf(@script) if @script != ""
141
148
  params[:_retry_on_conflict] = @retry_on_conflict
142
149
  end
150
+
143
151
  params
144
152
  end
145
153
 
data/lib/logstash/outputs/elasticsearch/common_configs.rb CHANGED
@@ -156,6 +156,9 @@ module LogStash; module Outputs; class ElasticSearch
156
156
  # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
157
157
  # for more info
158
158
  mod.config :retry_on_conflict, :validate => :number, :default => 1
159
+
160
+ # Set which ingest pipeline you wish to execute for an event
161
+ mod.config :pipeline, :validate => :string, :default => nil
159
162
  end
160
163
  end
161
164
  end end end
data/logstash-output-elasticsearch.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '2.6.2'
4
+ s.version = '2.7.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
data/spec/integration/outputs/pipeline_spec.rb ADDED
@@ -0,0 +1,75 @@
1
+ require_relative "../../../spec/es_spec_helper"
2
+
3
+ describe "Ingest pipeline execution behavior", :integration => true, :version_5x => true do
4
+ subject! do
5
+ require "logstash/outputs/elasticsearch"
6
+ settings = {
7
+ "hosts" => "#{get_host_port()}",
8
+ "pipeline" => "apache-logs"
9
+ }
10
+ next LogStash::Outputs::ElasticSearch.new(settings)
11
+ end
12
+
13
+ let(:ftw_client) { FTW::Agent.new }
14
+ let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
15
+ let(:apache_logs_pipeline) { '
16
+ {
17
+ "description" : "Pipeline to parse Apache logs",
18
+ "processors" : [
19
+ {
20
+ "grok": {
21
+ "field": "message",
22
+ "pattern": "%{COMBINEDAPACHELOG}"
23
+ }
24
+ }
25
+ ]
26
+ }'
27
+ }
28
+
29
+ before :each do
30
+ # Delete all templates first.
31
+ require "elasticsearch"
32
+
33
+ # Clean ES of data before we start.
34
+ @es = get_client
35
+ @es.indices.delete_template(:name => "*")
36
+
37
+ # This can fail if there are no indexes, ignore failure.
38
+ @es.indices.delete(:index => "*") rescue nil
39
+
40
+ # delete existing ingest pipeline
41
+ req = ftw_client.delete(ingest_url)
42
+ ftw_client.execute(req)
43
+
44
+ # register pipeline
45
+ req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
46
+ ftw_client.execute(req)
47
+
48
+ #TODO: Use esclient
49
+ #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
50
+
51
+ subject.register
52
+ subject.receive(LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"'))
53
+ subject.flush
54
+ @es.indices.refresh
55
+
56
+ #Wait or fail until everything's indexed.
57
+ Stud::try(20.times) do
58
+ r = @es.search
59
+ insist { r["hits"]["total"] } == 1
60
+ end
61
+ end
62
+
63
+ it "indexes using the proper pipeline" do
64
+ results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
65
+ insist { results["hits"]["total"] } == 1
66
+ insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
67
+ insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
68
+ insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
69
+ insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
70
+ insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
71
+ insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
72
+ insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
73
+ insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
74
+ end
75
+ end
data/spec/integration/outputs/templates_spec.rb CHANGED
@@ -73,7 +73,7 @@ describe "index template expected behavior", :integration => true do
73
73
  end
74
74
 
75
75
  it "make [geoip][location] a geo_point" do
76
- results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
76
+ results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
77
77
  insist { results["hits"]["total"] } == 1
78
78
  insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
79
79
  end
metadata CHANGED
@@ -1,39 +1,30 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.6.2
4
+ version: 2.7.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-20 00:00:00.000000000 Z
11
+ date: 2016-04-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: concurrent-ruby
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '>='
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - '>='
23
17
  - !ruby/object:Gem::Version
24
18
  version: '0'
19
+ name: concurrent-ruby
25
20
  prerelease: false
26
21
  type: :runtime
27
- - !ruby/object:Gem::Dependency
28
- name: elasticsearch
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
24
  - - '>='
32
25
  - !ruby/object:Gem::Version
33
- version: 1.0.13
34
- - - ~>
35
- - !ruby/object:Gem::Version
36
- version: '1.0'
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
37
28
  requirement: !ruby/object:Gem::Requirement
38
29
  requirements:
39
30
  - - '>='
@@ -42,18 +33,18 @@ dependencies:
42
33
  - - ~>
43
34
  - !ruby/object:Gem::Version
44
35
  version: '1.0'
36
+ name: elasticsearch
45
37
  prerelease: false
46
38
  type: :runtime
47
- - !ruby/object:Gem::Dependency
48
- name: stud
49
39
  version_requirements: !ruby/object:Gem::Requirement
50
40
  requirements:
51
41
  - - '>='
52
42
  - !ruby/object:Gem::Version
53
- version: 0.0.17
43
+ version: 1.0.13
54
44
  - - ~>
55
45
  - !ruby/object:Gem::Version
56
- version: '0.0'
46
+ version: '1.0'
47
+ - !ruby/object:Gem::Dependency
57
48
  requirement: !ruby/object:Gem::Requirement
58
49
  requirements:
59
50
  - - '>='
@@ -62,74 +53,74 @@ dependencies:
62
53
  - - ~>
63
54
  - !ruby/object:Gem::Version
64
55
  version: '0.0'
56
+ name: stud
65
57
  prerelease: false
66
58
  type: :runtime
67
- - !ruby/object:Gem::Dependency
68
- name: cabin
69
59
  version_requirements: !ruby/object:Gem::Requirement
70
60
  requirements:
61
+ - - '>='
62
+ - !ruby/object:Gem::Version
63
+ version: 0.0.17
71
64
  - - ~>
72
65
  - !ruby/object:Gem::Version
73
- version: '0.6'
66
+ version: '0.0'
67
+ - !ruby/object:Gem::Dependency
74
68
  requirement: !ruby/object:Gem::Requirement
75
69
  requirements:
76
70
  - - ~>
77
71
  - !ruby/object:Gem::Version
78
72
  version: '0.6'
73
+ name: cabin
79
74
  prerelease: false
80
75
  type: :runtime
81
- - !ruby/object:Gem::Dependency
82
- name: logstash-core-plugin-api
83
76
  version_requirements: !ruby/object:Gem::Requirement
84
77
  requirements:
85
78
  - - ~>
86
79
  - !ruby/object:Gem::Version
87
- version: '1.0'
80
+ version: '0.6'
81
+ - !ruby/object:Gem::Dependency
88
82
  requirement: !ruby/object:Gem::Requirement
89
83
  requirements:
90
84
  - - ~>
91
85
  - !ruby/object:Gem::Version
92
86
  version: '1.0'
87
+ name: logstash-core-plugin-api
93
88
  prerelease: false
94
89
  type: :runtime
95
- - !ruby/object:Gem::Dependency
96
- name: ftw
97
90
  version_requirements: !ruby/object:Gem::Requirement
98
91
  requirements:
99
92
  - - ~>
100
93
  - !ruby/object:Gem::Version
101
- version: 0.0.42
94
+ version: '1.0'
95
+ - !ruby/object:Gem::Dependency
102
96
  requirement: !ruby/object:Gem::Requirement
103
97
  requirements:
104
98
  - - ~>
105
99
  - !ruby/object:Gem::Version
106
100
  version: 0.0.42
101
+ name: ftw
107
102
  prerelease: false
108
103
  type: :development
109
- - !ruby/object:Gem::Dependency
110
- name: logstash-codec-plain
111
104
  version_requirements: !ruby/object:Gem::Requirement
112
105
  requirements:
113
- - - '>='
106
+ - - ~>
114
107
  - !ruby/object:Gem::Version
115
- version: '0'
108
+ version: 0.0.42
109
+ - !ruby/object:Gem::Dependency
116
110
  requirement: !ruby/object:Gem::Requirement
117
111
  requirements:
118
112
  - - '>='
119
113
  - !ruby/object:Gem::Version
120
114
  version: '0'
115
+ name: logstash-codec-plain
121
116
  prerelease: false
122
117
  type: :development
123
- - !ruby/object:Gem::Dependency
124
- name: manticore
125
118
  version_requirements: !ruby/object:Gem::Requirement
126
119
  requirements:
127
120
  - - '>='
128
121
  - !ruby/object:Gem::Version
129
- version: 0.5.4
130
- - - <
131
- - !ruby/object:Gem::Version
132
- version: 1.0.0
122
+ version: '0'
123
+ - !ruby/object:Gem::Dependency
133
124
  requirement: !ruby/object:Gem::Requirement
134
125
  requirements:
135
126
  - - '>='
@@ -138,50 +129,59 @@ dependencies:
138
129
  - - <
139
130
  - !ruby/object:Gem::Version
140
131
  version: 1.0.0
132
+ name: manticore
141
133
  prerelease: false
142
134
  type: :runtime
143
- - !ruby/object:Gem::Dependency
144
- name: logstash-devutils
145
135
  version_requirements: !ruby/object:Gem::Requirement
146
136
  requirements:
147
137
  - - '>='
148
138
  - !ruby/object:Gem::Version
149
- version: '0'
139
+ version: 0.5.4
140
+ - - <
141
+ - !ruby/object:Gem::Version
142
+ version: 1.0.0
143
+ - !ruby/object:Gem::Dependency
150
144
  requirement: !ruby/object:Gem::Requirement
151
145
  requirements:
152
146
  - - '>='
153
147
  - !ruby/object:Gem::Version
154
148
  version: '0'
149
+ name: logstash-devutils
155
150
  prerelease: false
156
151
  type: :development
157
- - !ruby/object:Gem::Dependency
158
- name: longshoreman
159
152
  version_requirements: !ruby/object:Gem::Requirement
160
153
  requirements:
161
154
  - - '>='
162
155
  - !ruby/object:Gem::Version
163
156
  version: '0'
157
+ - !ruby/object:Gem::Dependency
164
158
  requirement: !ruby/object:Gem::Requirement
165
159
  requirements:
166
160
  - - '>='
167
161
  - !ruby/object:Gem::Version
168
162
  version: '0'
163
+ name: longshoreman
169
164
  prerelease: false
170
165
  type: :development
171
- - !ruby/object:Gem::Dependency
172
- name: flores
173
166
  version_requirements: !ruby/object:Gem::Requirement
174
167
  requirements:
175
168
  - - '>='
176
169
  - !ruby/object:Gem::Version
177
170
  version: '0'
171
+ - !ruby/object:Gem::Dependency
178
172
  requirement: !ruby/object:Gem::Requirement
179
173
  requirements:
180
174
  - - '>='
181
175
  - !ruby/object:Gem::Version
182
176
  version: '0'
177
+ name: flores
183
178
  prerelease: false
184
179
  type: :development
180
+ version_requirements: !ruby/object:Gem::Requirement
181
+ requirements:
182
+ - - '>='
183
+ - !ruby/object:Gem::Version
184
+ version: '0'
185
185
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
186
186
  email: info@elastic.co
187
187
  executables: []
@@ -210,6 +210,7 @@ files:
210
210
  - spec/integration/outputs/create_spec.rb
211
211
  - spec/integration/outputs/index_spec.rb
212
212
  - spec/integration/outputs/parent_spec.rb
213
+ - spec/integration/outputs/pipeline_spec.rb
213
214
  - spec/integration/outputs/retry_spec.rb
214
215
  - spec/integration/outputs/routing_spec.rb
215
216
  - spec/integration/outputs/secure_spec.rb
@@ -243,7 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
243
244
  version: '0'
244
245
  requirements: []
245
246
  rubyforge_project:
246
- rubygems_version: 2.4.8
247
+ rubygems_version: 2.4.5
247
248
  signing_key:
248
249
  specification_version: 4
249
250
  summary: Logstash Output to Elasticsearch
@@ -255,6 +256,7 @@ test_files:
255
256
  - spec/integration/outputs/create_spec.rb
256
257
  - spec/integration/outputs/index_spec.rb
257
258
  - spec/integration/outputs/parent_spec.rb
259
+ - spec/integration/outputs/pipeline_spec.rb
258
260
  - spec/integration/outputs/retry_spec.rb
259
261
  - spec/integration/outputs/routing_spec.rb
260
262
  - spec/integration/outputs/secure_spec.rb