logstash-filter-elasticsearch 2.0.4 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (13) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -4
  3. data/Gemfile +2 -1
  4. data/LICENSE +1 -1
  5. data/README.md +12 -3
  6. data/lib/logstash/filters/elasticsearch.rb +39 -37
  7. data/lib/logstash/filters/elasticsearch/client.rb +35 -0
  8. data/logstash-filter-elasticsearch.gemspec +2 -2
  9. data/spec/filters/elasticsearch_spec.rb +66 -1
  10. data/spec/filters/fixtures/request_x_1.json +62 -0
  11. data/spec/filters/fixtures/request_x_10.json +500 -0
  12. data/spec/filters/integration/elasticsearch_spec.rb +44 -0
  13. metadata +10 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a45e372b9a69c832048cf4c3ce538d1c71e4e8ac
4
- data.tar.gz: a91ec84ccc0b978d4c0d4d3a7f7486d5e1c2ce2e
3
+ metadata.gz: cfc8b5e9cf33d9d7ef26f24fa0692a8e64119c17
4
+ data.tar.gz: 84e9261001fa6a28fd111391fcb23b36c41ce834
5
5
  SHA512:
6
- metadata.gz: 1dc53b2ac23c20d84311bb4164b2933a86625d089eb0f3e38805628d2ff805b0ced72e8423cc79ceb0907d27c61d4ed1ca9647345ec835ce055697ca9d636695
7
- data.tar.gz: aa1267575f30dafa1f3cfc4692ab57c7fc44b2cf175424773a093a926938dda62b8e4cb1ee9d9ea8a62ac2b7f22e7fbb03c06d981f530aaedc31148a84565739
6
+ metadata.gz: 853a1678509361a7eedd80a0fc8264ac7184b0aaf6358cf0373ee0a9783af96620bf873cd12a5d08ba0c9bb5f8966883d68f36fc0c228e64a01b82ed8292134f
7
+ data.tar.gz: bf37c8e634112d4d5b8cb86cd018cd200e6f91fa11bffabc0fec9e5f38f6dd00eb0404ae4d8fe99fc79ee68df47b5f912a91f4d594b2c54e54169ec9564eb260
data/CHANGELOG.md CHANGED
@@ -1,12 +1,18 @@
1
- # 2.0.4
1
+ ## 2.1.0
2
+ - Improved the configuration options to be more easy to understand and
3
+ match what the expectations are from the documentation.
4
+ - Initial refactoring to include later one a common client for all the
5
+ ES plugins.
6
+ - Adding support for having an index in the query pattern.
7
+ - Improved documentation.
8
+ - Added intitial integration and unit tests.
9
+ ## 2.0.4
2
10
  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
3
- # 2.0.3
11
+ ## 2.0.3
4
12
  - New dependency requirements for logstash-core for the 5.0 release
5
13
  ## 2.0.0
6
14
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
7
15
  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
8
16
  - Dependency on logstash-core update to 2.0
9
-
10
17
  ## 0.1.6
11
-
12
18
  - removed require statement for a file that is no longer present in logstash-core.
data/Gemfile CHANGED
@@ -1,2 +1,3 @@
1
1
  source 'https://rubygems.org'
2
- gemspec
2
+ gemspec
3
+ gem "pry", :group => :development
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
1
+ Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
1
  # Logstash Plugin
2
2
 
3
- [![Build
4
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-elasticsearch-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-elasticsearch-unit/)
3
+ [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch)
5
4
 
6
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
7
6
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
56
55
  ```
57
56
  - Install plugin
58
57
  ```sh
58
+ # Logstash 2.3 and higher
59
+ bin/logstash-plugin install --no-verify
60
+
61
+ # Prior to Logstash 2.3
59
62
  bin/plugin install --no-verify
63
+
60
64
  ```
61
65
  - Run Logstash with your plugin
62
66
  ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
74
78
  ```
75
79
  - Install the plugin from the Logstash home
76
80
  ```sh
77
- bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
81
+ # Logstash 2.3 and higher
82
+ bin/logstash-plugin install --no-verify
83
+
84
+ # Prior to Logstash 2.3
85
+ bin/plugin install --no-verify
86
+
78
87
  ```
79
88
  - Start Logstash and proceed to test the plugin
80
89
 
data/lib/logstash/filters/elasticsearch.rb CHANGED
@@ -1,6 +1,7 @@
1
+ # encoding: utf-8
1
2
  require "logstash/filters/base"
2
3
  require "logstash/namespace"
3
- require "base64"
4
+ require_relative "elasticsearch/client"
4
5
 
5
6
 
6
7
  # Search elasticsearch for a previous log event and copy some fields from it
@@ -15,7 +16,7 @@ require "base64"
15
16
  # elasticsearch {
16
17
  # hosts => ["es-server"]
17
18
  # query => "type:start AND operation:%{[opid]}"
18
- # fields => ["@timestamp", "started"]
19
+ # fields => [["@timestamp", "started"]]
19
20
  # }
20
21
  #
21
22
  # date {
@@ -32,9 +33,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
32
33
  config_name "elasticsearch"
33
34
 
34
35
  # List of elasticsearch hosts to use for querying.
35
- config :hosts, :validate => :array
36
+ config :hosts, :validate => :array, :default => [ "localhost:9200" ]
37
+
38
+ # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
39
+ config :index, :validate => :string, :default => ""
36
40
 
37
- # Elasticsearch query string
41
+ # Elasticsearch query string. Read the Elasticsearch query string documentation
42
+ # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
38
43
  config :query, :validate => :string
39
44
 
40
45
  # Comma-delimited list of `<field>:<direction>` pairs that define the sort order
@@ -55,49 +60,46 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
55
60
  # SSL Certificate Authority file
56
61
  config :ca_file, :validate => :path
57
62
 
63
+ # Whether results should be sorted or not
64
+ config :enable_sort, :validate => :boolean, :default => true
58
65
 
59
- public
60
- def register
61
- require "elasticsearch"
62
-
63
- transport_options = {}
64
-
65
- if @user && @password
66
- token = Base64.strict_encode64("#{@user}:#{@password.value}")
67
- transport_options[:headers] = { Authorization: "Basic #{token}" }
68
- end
69
-
70
- hosts = if @ssl then
71
- @hosts.map {|h| { host: h, scheme: 'https' } }
72
- else
73
- @hosts
74
- end
66
+ # How many results to return
67
+ config :result_size, :validate => :number, :default => 1
75
68
 
76
- if @ssl && @ca_file
77
- transport_options[:ssl] = { ca_file: @ca_file }
78
- end
69
+ # Tags the event on failure to look up geo information. This can be used in later analysis.
70
+ config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
79
71
 
80
- @logger.info("New ElasticSearch filter", :hosts => hosts)
81
- @client = Elasticsearch::Client.new hosts: hosts, transport_options: transport_options
72
+ def register
73
+ options = {
74
+ :ssl => @ssl,
75
+ :hosts => @hosts,
76
+ :ca_file => @ca_file,
77
+ :logger => @logger,
78
+ :index => @index
79
+ }
80
+ @client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
82
81
  end # def register
83
82
 
84
- public
85
83
  def filter(event)
86
-
87
-
88
84
  begin
89
85
  query_str = event.sprintf(@query)
90
-
91
- results = @client.search q: query_str, sort: @sort, size: 1
92
-
93
- @fields.each do |old, new|
94
- event[new] = results['hits']['hits'][0]['_source'][old]
86
+ params = { :q => query_str, :size => result_size }
87
+ params[:sort] = @sort if @enable_sort
88
+ results = @client.search(params)
89
+
90
+ @fields.each do |old_key, new_key|
91
+ if !results['hits']['hits'].empty?
92
+ set = []
93
+ results["hits"]["hits"].to_a.each do |doc|
94
+ set << doc["_source"][old_key]
95
+ end
96
+ event[new_key] = ( set.count > 1 ? set : set.first)
97
+ end
95
98
  end
96
-
97
- filter_matched(event)
98
99
  rescue => e
99
- @logger.warn("Failed to query elasticsearch for previous event",
100
- :query => query_str, :event => event, :error => e)
100
+ @logger.warn("Failed to query elasticsearch for previous event", :index, @index, :query => query_str, :event => event, :error => e)
101
+ @tag_on_failure.each{|tag| event.tag(tag)}
101
102
  end
103
+ filter_matched(event)
102
104
  end # def filter
103
105
  end # class LogStash::Filters::Elasticsearch
data/lib/logstash/filters/elasticsearch/client.rb ADDED
@@ -0,0 +1,35 @@
1
+ # encoding: utf-8
2
+ require "elasticsearch"
3
+ require "base64"
4
+
5
+ module LogStash
6
+ module Filters
7
+ class ElasticsearchClient
8
+
9
+ attr_reader :client
10
+
11
+ def initialize(user, password, options={})
12
+ ssl = options.fetch(:ssh, false)
13
+ hosts = options[:hosts]
14
+ @logger = options[:logger]
15
+
16
+ transport_options = {}
17
+ if user && password
18
+ token = ::Base64.strict_encode64("#{user}:#{password.value}")
19
+ transport_options[:headers] = { Authorization: "Basic #{token}" }
20
+ end
21
+
22
+ host.map! {|h| { host: h, scheme: 'https' } } if ssl
23
+ transport_options[:ssl] = { ca_file: options[:ca_file] } if ssl && options[:ca_file]
24
+
25
+ @logger.info("New ElasticSearch filter", :hosts => hosts)
26
+ @client = ::Elasticsearch::Client.new(index: options[:index], hosts: hosts, transport_options: transport_options)
27
+ end
28
+
29
+ def search(params)
30
+ @client.search(params)
31
+ end
32
+
33
+ end
34
+ end
35
+ end
data/logstash-filter-elasticsearch.gemspec CHANGED
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-elasticsearch'
4
- s.version = '2.0.4'
4
+ s.version = '2.1.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
7
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elastic"]
9
9
  s.email = 'info@elastic.co'
10
10
  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
data/spec/filters/elasticsearch_spec.rb CHANGED
@@ -1,8 +1,8 @@
1
1
  # encoding: utf-8
2
-
3
2
  require "logstash/devutils/rspec/spec_helper"
4
3
  require "logstash/plugin"
5
4
  require "logstash/filters/elasticsearch"
5
+ require "logstash/json"
6
6
 
7
7
  describe LogStash::Filters::Elasticsearch do
8
8
 
@@ -15,4 +15,69 @@ describe LogStash::Filters::Elasticsearch do
15
15
  end
16
16
  end
17
17
 
18
+ describe "data fetch" do
19
+ let(:config) do
20
+ {
21
+ "hosts" => ["localhost:9200"],
22
+ "query" => "response: 404",
23
+ "fields" => [ ["response", "code"] ],
24
+ }
25
+ end
26
+ let(:plugin) { described_class.new(config) }
27
+ let(:event) { LogStash::Event.new({}) }
28
+
29
+ let(:response) do
30
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
31
+ end
32
+
33
+ let(:client) { double(:client) }
34
+
35
+ before(:each) do
36
+ allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
37
+ allow(client).to receive(:search).and_return(response)
38
+ plugin.register
39
+ end
40
+
41
+ it "should enhance the current event with new data" do
42
+ plugin.filter(event)
43
+ expect(event["code"]).to eq(404)
44
+ end
45
+
46
+ context "when asking for more than one result" do
47
+
48
+ let(:config) do
49
+ {
50
+ "hosts" => ["localhost:9200"],
51
+ "query" => "response: 404",
52
+ "fields" => [ ["response", "code"] ],
53
+ "result_size" => 10
54
+ }
55
+ end
56
+
57
+ let(:response) do
58
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
59
+ end
60
+
61
+ it "should enhance the current event with new data" do
62
+ plugin.filter(event)
63
+ expect(event["code"]).to eq([404]*10)
64
+ end
65
+ end
66
+
67
+ context "if something wrong happen during connection" do
68
+
69
+ before(:each) do
70
+ allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
71
+ allow(client).to receive(:search).and_raise("connection exception")
72
+ plugin.register
73
+ end
74
+
75
+ it "tag the event as something happened, but still deliver it" do
76
+ expect(plugin.logger).to receive(:warn)
77
+ plugin.filter(event)
78
+ expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
79
+ end
80
+ end
81
+ end
82
+
18
83
  end
data/spec/filters/fixtures/request_x_1.json ADDED
@@ -0,0 +1,62 @@
1
+ {
2
+ "took": 49,
3
+ "timed_out": false,
4
+ "_shards": {
5
+ "total": 155,
6
+ "successful": 155,
7
+ "failed": 0
8
+ },
9
+ "hits": {
10
+ "total": 13476,
11
+ "max_score": 1,
12
+ "hits": [{
13
+ "_index": "logstash-2014.08.26",
14
+ "_type": "logs",
15
+ "_id": "AVVY76L_AW7v0kX8KXo4",
16
+ "_score": 1,
17
+ "_source": {
18
+ "request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
19
+ "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
20
+ "geoip": {
21
+ "timezone": "America/Los_Angeles",
22
+ "ip": "66.249.73.185",
23
+ "latitude": 37.386,
24
+ "continent_code": "NA",
25
+ "city_name": "Mountain View",
26
+ "country_code2": "US",
27
+ "country_name": "United States",
28
+ "dma_code": 807,
29
+ "country_code3": "US",
30
+ "region_name": "California",
31
+ "location": [-122.0838,
32
+ 37.386
33
+ ],
34
+ "postal_code": "94035",
35
+ "longitude": -122.0838,
36
+ "region_code": "CA"
37
+ },
38
+ "auth": "-",
39
+ "ident": "-",
40
+ "verb": "GET",
41
+ "useragent": {
42
+ "os": "Other",
43
+ "major": "2",
44
+ "minor": "1",
45
+ "name": "Googlebot",
46
+ "os_name": "Other",
47
+ "device": "Spider"
48
+ },
49
+ "message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
50
+ "referrer": "\"-\"",
51
+ "@timestamp": "2014-08-26T21:22:13.000Z",
52
+ "response": 404,
53
+ "bytes": 294,
54
+ "clientip": "66.249.73.185",
55
+ "@version": "1",
56
+ "host": "skywalker",
57
+ "httpversion": "1.1",
58
+ "timestamp": "26/Aug/2014:21:22:13 +0000"
59
+ }
60
+ }]
61
+ }
62
+ }
data/spec/filters/fixtures/request_x_10.json ADDED
@@ -0,0 +1,500 @@
1
+ {
2
+ "took": 49,
3
+ "timed_out": false,
4
+ "_shards": {
5
+ "total": 155,
6
+ "successful": 155,
7
+ "failed": 0
8
+ },
9
+ "hits": {
10
+ "total": 13476,
11
+ "max_score": 1,
12
+ "hits": [{
13
+ "_index": "logstash-2014.08.26",
14
+ "_type": "logs",
15
+ "_id": "AVVY76L_AW7v0kX8KXo4",
16
+ "_score": 1,
17
+ "_source": {
18
+ "request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
19
+ "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
20
+ "geoip": {
21
+ "timezone": "America/Los_Angeles",
22
+ "ip": "66.249.73.185",
23
+ "latitude": 37.386,
24
+ "continent_code": "NA",
25
+ "city_name": "Mountain View",
26
+ "country_code2": "US",
27
+ "country_name": "United States",
28
+ "dma_code": 807,
29
+ "country_code3": "US",
30
+ "region_name": "California",
31
+ "location": [-122.0838,
32
+ 37.386
33
+ ],
34
+ "postal_code": "94035",
35
+ "longitude": -122.0838,
36
+ "region_code": "CA"
37
+ },
38
+ "auth": "-",
39
+ "ident": "-",
40
+ "verb": "GET",
41
+ "useragent": {
42
+ "os": "Other",
43
+ "major": "2",
44
+ "minor": "1",
45
+ "name": "Googlebot",
46
+ "os_name": "Other",
47
+ "device": "Spider"
48
+ },
49
+ "message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
50
+ "referrer": "\"-\"",
51
+ "@timestamp": "2014-08-26T21:22:13.000Z",
52
+ "response": 404,
53
+ "bytes": 294,
54
+ "clientip": "66.249.73.185",
55
+ "@version": "1",
56
+ "host": "skywalker",
57
+ "httpversion": "1.1",
58
+ "timestamp": "26/Aug/2014:21:22:13 +0000"
59
+ }
60
+ }, {
61
+ "_index": "logstash-2014.08.26",
62
+ "_type": "logs",
63
+ "_id": "AVVY76eJAW7v0kX8KXtH",
64
+ "_score": 1,
65
+ "_source": {
66
+ "request": "/presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif",
67
+ "agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1\"",
68
+ "geoip": {
69
+ "timezone": "Asia/Shanghai",
70
+ "ip": "111.199.235.239",
71
+ "latitude": 39.9289,
72
+ "continent_code": "AS",
73
+ "city_name": "Beijing",
74
+ "country_code2": "CN",
75
+ "country_name": "China",
76
+ "dma_code": null,
77
+ "country_code3": "CN",
78
+ "region_name": "Beijing",
79
+ "location": [
80
+ 116.3883,
81
+ 39.9289
82
+ ],
83
+ "postal_code": null,
84
+ "longitude": 116.3883,
85
+ "region_code": "11"
86
+ },
87
+ "auth": "-",
88
+ "ident": "-",
89
+ "verb": "GET",
90
+ "useragent": {
91
+ "patch": "5",
92
+ "os": "Mac OS X 10.8.5",
93
+ "major": "6",
94
+ "minor": "0",
95
+ "os_minor": "8",
96
+ "os_major": "10",
97
+ "name": "Safari",
98
+ "os_name": "Mac OS X",
99
+ "device": "Other"
100
+ },
101
+ "message": "111.199.235.239 - - [26/Aug/2014:22:06:06 +0000] \"GET /presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif HTTP/1.1\" 404 364 \"http://semicomplete.com/presentations/logstash-puppetconf-2012/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1\"",
102
+ "referrer": "\"http://semicomplete.com/presentations/logstash-puppetconf-2012/\"",
103
+ "@timestamp": "2014-08-26T22:06:06.000Z",
104
+ "response": 404,
105
+ "bytes": 364,
106
+ "clientip": "111.199.235.239",
107
+ "@version": "1",
108
+ "host": "skywalker",
109
+ "httpversion": "1.1",
110
+ "timestamp": "26/Aug/2014:22:06:06 +0000"
111
+ }
112
+ }, {
113
+ "_index": "logstash-2014.08.26",
114
+ "_type": "logs",
115
+ "_id": "AVVY76eJAW7v0kX8KXtf",
116
+ "_score": 1,
117
+ "_source": {
118
+ "request": "/files/logstash/logstash-1.3.2-monolithic.jar",
119
+ "agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
120
+ "geoip": {
121
+ "timezone": "America/Los_Angeles",
122
+ "ip": "208.91.156.11",
123
+ "latitude": 34.0486,
124
+ "continent_code": "NA",
125
+ "city_name": "Los Angeles",
126
+ "country_code2": "US",
127
+ "country_name": "United States",
128
+ "dma_code": 803,
129
+ "country_code3": "US",
130
+ "region_name": "California",
131
+ "location": [-118.4424,
132
+ 34.0486
133
+ ],
134
+ "postal_code": "90025",
135
+ "longitude": -118.4424,
136
+ "region_code": "CA"
137
+ },
138
+ "auth": "-",
139
+ "ident": "-",
140
+ "verb": "GET",
141
+ "useragent": {
142
+ "os": "Other",
143
+ "name": "Other",
144
+ "os_name": "Other",
145
+ "device": "Other"
146
+ },
147
+ "message": "208.91.156.11 - - [26/Aug/2014:22:12:14 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
148
+ "referrer": "\"-\"",
149
+ "@timestamp": "2014-08-26T22:12:14.000Z",
150
+ "response": 404,
151
+ "bytes": 324,
152
+ "clientip": "208.91.156.11",
153
+ "@version": "1",
154
+ "host": "skywalker",
155
+ "httpversion": "1.1",
156
+ "timestamp": "26/Aug/2014:22:12:14 +0000"
157
+ }
158
+ }, {
159
+ "_index": "logstash-2014.08.26",
160
+ "_type": "logs",
161
+ "_id": "AVVY761xAW7v0kX8KXvw",
162
+ "_score": 1,
163
+ "_source": {
164
+ "request": "/files/logstash/logstash-1.3.2-monolithic.jar",
165
+ "agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
166
+ "geoip": {
167
+ "timezone": "America/Los_Angeles",
168
+ "ip": "208.91.156.11",
169
+ "latitude": 34.0486,
170
+ "continent_code": "NA",
171
+ "city_name": "Los Angeles",
172
+ "country_code2": "US",
173
+ "country_name": "United States",
174
+ "dma_code": 803,
175
+ "country_code3": "US",
176
+ "region_name": "California",
177
+ "location": [-118.4424,
178
+ 34.0486
179
+ ],
180
+ "postal_code": "90025",
181
+ "longitude": -118.4424,
182
+ "region_code": "CA"
183
+ },
184
+ "auth": "-",
185
+ "ident": "-",
186
+ "verb": "GET",
187
+ "useragent": {
188
+ "os": "Other",
189
+ "name": "Other",
190
+ "os_name": "Other",
191
+ "device": "Other"
192
+ },
193
+ "message": "208.91.156.11 - - [26/Aug/2014:22:42:22 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
194
+ "referrer": "\"-\"",
195
+ "@timestamp": "2014-08-26T22:42:22.000Z",
196
+ "response": 404,
197
+ "bytes": 324,
198
+ "clientip": "208.91.156.11",
199
+ "@version": "1",
200
+ "host": "skywalker",
201
+ "httpversion": "1.1",
202
+ "timestamp": "26/Aug/2014:22:42:22 +0000"
203
+ }
204
+ }, {
205
+ "_index": "logstash-2014.08.26",
206
+ "_type": "logs",
207
+ "_id": "AVVY77AwAW7v0kX8KXx8",
208
+ "_score": 1,
209
+ "_source": {
210
+ "request": "/wp-login.php",
211
+ "agent": "\"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13\"",
212
+ "geoip": {
213
+ "timezone": "Europe/Rome",
214
+ "ip": "195.250.34.144",
215
+ "latitude": 43.4995,
216
+ "continent_code": "EU",
217
+ "city_name": "Arezzo",
218
+ "country_code2": "IT",
219
+ "country_name": "Italy",
220
+ "dma_code": null,
221
+ "country_code3": "IT",
222
+ "region_name": "Province of Arezzo",
223
+ "location": [
224
+ 11.9109,
225
+ 43.4995
226
+ ],
227
+ "postal_code": "52100",
228
+ "longitude": 11.9109,
229
+ "region_code": "AR"
230
+ },
231
+ "auth": "-",
232
+ "ident": "-",
233
+ "verb": "GET",
234
+ "useragent": {
235
+ "patch": "1290",
236
+ "os": "Windows 7",
237
+ "major": "24",
238
+ "minor": "0",
239
+ "name": "Chrome",
240
+ "os_name": "Windows 7",
241
+ "device": "Other"
242
+ },
243
+ "message": "195.250.34.144 - - [26/Aug/2014:23:40:50 +0000] \"GET /wp-login.php HTTP/1.1\" 404 292 \"-\" \"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13\"",
244
+ "referrer": "\"-\"",
245
+ "@timestamp": "2014-08-26T23:40:50.000Z",
246
+ "response": 404,
247
+ "bytes": 292,
248
+ "clientip": "195.250.34.144",
249
+ "@version": "1",
250
+ "host": "skywalker",
251
+ "httpversion": "1.1",
252
+ "timestamp": "26/Aug/2014:23:40:50 +0000"
253
+ }
254
+ }, {
255
+ "_index": "logstash-2014.08.26",
256
+ "_type": "logs",
257
+ "_id": "AVVY77AwAW7v0kX8KXyB",
258
+ "_score": 1,
259
+ "_source": {
260
+ "request": "/presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif",
261
+ "agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/6.1.1 Safari/537.73.11\"",
262
+ "geoip": {
263
+ "timezone": "Asia/Kolkata",
264
+ "ip": "122.166.142.108",
265
+ "latitude": 12.9833,
266
+ "continent_code": "AS",
267
+ "city_name": "Bengaluru",
268
+ "country_code2": "IN",
269
+ "country_name": "India",
270
+ "dma_code": null,
271
+ "country_code3": "IN",
272
+ "region_name": "Karnataka",
273
+ "location": [
274
+ 77.5833,
275
+ 12.9833
276
+ ],
277
+ "postal_code": null,
278
+ "longitude": 77.5833,
279
+ "region_code": "KA"
280
+ },
281
+ "auth": "-",
282
+ "ident": "-",
283
+ "verb": "GET",
284
+ "useragent": {
285
+ "patch": "1",
286
+ "os": "Mac OS X 10.8.5",
287
+ "major": "6",
288
+ "minor": "1",
289
+ "os_minor": "8",
290
+ "os_major": "10",
291
+ "name": "Safari",
292
+ "os_name": "Mac OS X",
293
+ "device": "Other"
294
+ },
295
+ "message": "122.166.142.108 - - [26/Aug/2014:23:41:19 +0000] \"GET /presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif HTTP/1.1\" 404 364 \"http://semicomplete.com/presentations/logstash-puppetconf-2012/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/6.1.1 Safari/537.73.11\"",
296
+ "referrer": "\"http://semicomplete.com/presentations/logstash-puppetconf-2012/\"",
297
+ "@timestamp": "2014-08-26T23:41:19.000Z",
298
+ "response": 404,
299
+ "bytes": 364,
300
+ "clientip": "122.166.142.108",
301
+ "@version": "1",
302
+ "host": "skywalker",
303
+ "httpversion": "1.1",
304
+ "timestamp": "26/Aug/2014:23:41:19 +0000"
305
+ }
306
+ }, {
307
+ "_index": "logstash-2014.08.26",
308
+ "_type": "logs",
309
+ "_id": "AVVY77NUAW7v0kX8KX0s",
310
+ "_score": 1,
311
+ "_source": {
312
+ "request": "/projects/xdotool%3E",
313
+ "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
314
+ "geoip": {
315
+ "timezone": "America/Los_Angeles",
316
+ "ip": "66.249.73.135",
317
+ "latitude": 37.386,
318
+ "continent_code": "NA",
319
+ "city_name": "Mountain View",
320
+ "country_code2": "US",
321
+ "country_name": "United States",
322
+ "dma_code": 807,
323
+ "country_code3": "US",
324
+ "region_name": "California",
325
+ "location": [-122.0838,
326
+ 37.386
327
+ ],
328
+ "postal_code": "94035",
329
+ "longitude": -122.0838,
330
+ "region_code": "CA"
331
+ },
332
+ "auth": "-",
333
+ "ident": "-",
334
+ "verb": "GET",
335
+ "useragent": {
336
+ "os": "Other",
337
+ "major": "2",
338
+ "minor": "1",
339
+ "name": "Googlebot",
340
+ "os_name": "Other",
341
+ "device": "Spider"
342
+ },
343
+ "message": "66.249.73.135 - - [26/Aug/2014:23:25:32 +0000] \"GET /projects/xdotool%3E HTTP/1.1\" 404 7861 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
344
+ "referrer": "\"-\"",
345
+ "@timestamp": "2014-08-26T23:25:32.000Z",
346
+ "response": 404,
347
+ "bytes": 7861,
348
+ "clientip": "66.249.73.135",
349
+ "@version": "1",
350
+ "host": "skywalker",
351
+ "httpversion": "1.1",
352
+ "timestamp": "26/Aug/2014:23:25:32 +0000"
353
+ }
354
+ }, {
355
+ "_index": "logstash-2014.08.27",
356
+ "_type": "logs",
357
+ "_id": "AVVY77vzAW7v0kX8KX5_",
358
+ "_score": 1,
359
+ "_source": {
360
+ "request": "/wp-login.php?action=register",
361
+ "agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0\"",
362
+ "geoip": {
363
+ "timezone": "America/Chicago",
364
+ "ip": "198.143.145.210",
365
+ "latitude": 41.8825,
366
+ "continent_code": "NA",
367
+ "city_name": "Chicago",
368
+ "country_code2": "US",
369
+ "country_name": "United States",
370
+ "dma_code": 602,
371
+ "country_code3": "US",
372
+ "region_name": "Illinois",
373
+ "location": [-87.6441,
374
+ 41.8825
375
+ ],
376
+ "postal_code": "60661",
377
+ "longitude": -87.6441,
378
+ "region_code": "IL"
379
+ },
380
+ "auth": "-",
381
+ "ident": "-",
382
+ "verb": "GET",
383
+ "useragent": {
384
+ "os": "Mac OS X 10.7",
385
+ "major": "21",
386
+ "minor": "0",
387
+ "os_minor": "7",
388
+ "os_major": "10",
389
+ "name": "Firefox",
390
+ "os_name": "Mac OS X",
391
+ "device": "Other"
392
+ },
393
+ "message": "198.143.145.210 - - [27/Aug/2014:01:30:10 +0000] \"GET /wp-login.php?action=register HTTP/1.0\" 404 296 \"http://www.semicomplete.com/misc/sample.log\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0\"",
394
+ "referrer": "\"http://www.semicomplete.com/misc/sample.log\"",
395
+ "@timestamp": "2014-08-27T01:30:10.000Z",
396
+ "response": 404,
397
+ "bytes": 296,
398
+ "clientip": "198.143.145.210",
399
+ "@version": "1",
400
+ "host": "skywalker",
401
+ "httpversion": "1.0",
402
+ "timestamp": "27/Aug/2014:01:30:10 +0000"
403
+ }
404
+ }, {
405
+ "_index": "logstash-2014.08.27",
406
+ "_type": "logs",
407
+ "_id": "AVVY77vzAW7v0kX8KX6w",
408
+ "_score": 1,
409
+ "_source": {
410
+ "request": "/projects/securitrack/config.xsl",
411
+ "agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
412
+ "geoip": {
413
+ "timezone": "America/Los_Angeles",
414
+ "ip": "66.249.73.135",
415
+ "latitude": 37.386,
416
+ "continent_code": "NA",
417
+ "city_name": "Mountain View",
418
+ "country_code2": "US",
419
+ "country_name": "United States",
420
+ "dma_code": 807,
421
+ "country_code3": "US",
422
+ "region_name": "California",
423
+ "location": [-122.0838,
424
+ 37.386
425
+ ],
426
+ "postal_code": "94035",
427
+ "longitude": -122.0838,
428
+ "region_code": "CA"
429
+ },
430
+ "auth": "-",
431
+ "ident": "-",
432
+ "verb": "GET",
433
+ "useragent": {
434
+ "os": "Other",
435
+ "major": "2",
436
+ "minor": "1",
437
+ "name": "Googlebot",
438
+ "os_name": "Other",
439
+ "device": "Spider"
440
+ },
441
+ "message": "66.249.73.135 - - [27/Aug/2014:01:40:51 +0000] \"GET /projects/securitrack/config.xsl HTTP/1.1\" 404 315 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
442
+ "referrer": "\"-\"",
443
+ "@timestamp": "2014-08-27T01:40:51.000Z",
444
+ "response": 404,
445
+ "bytes": 315,
446
+ "clientip": "66.249.73.135",
447
+ "@version": "1",
448
+ "host": "skywalker",
449
+ "httpversion": "1.1",
450
+ "timestamp": "27/Aug/2014:01:40:51 +0000"
451
+ }
452
+ }, {
453
+ "_index": "logstash-2014.08.27",
454
+ "_type": "logs",
455
+ "_id": "AVVY78FiAW7v0kX8KYBM",
456
+ "_score": 1,
457
+ "_source": {
458
+ "request": "/files/logstash/logstash-1.3.2-monolithic.jar",
459
+ "agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
460
+ "geoip": {
461
+ "timezone": "America/Los_Angeles",
462
+ "ip": "208.91.156.11",
463
+ "latitude": 34.0486,
464
+ "continent_code": "NA",
465
+ "city_name": "Los Angeles",
466
+ "country_code2": "US",
467
+ "country_name": "United States",
468
+ "dma_code": 803,
469
+ "country_code3": "US",
470
+ "region_name": "California",
471
+ "location": [-118.4424,
472
+ 34.0486
473
+ ],
474
+ "postal_code": "90025",
475
+ "longitude": -118.4424,
476
+ "region_code": "CA"
477
+ },
478
+ "auth": "-",
479
+ "ident": "-",
480
+ "verb": "GET",
481
+ "useragent": {
482
+ "os": "Other",
483
+ "name": "Other",
484
+ "os_name": "Other",
485
+ "device": "Other"
486
+ },
487
+ "message": "208.91.156.11 - - [27/Aug/2014:02:44:04 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
488
+ "referrer": "\"-\"",
489
+ "@timestamp": "2014-08-27T02:44:04.000Z",
490
+ "response": 404,
491
+ "bytes": 324,
492
+ "clientip": "208.91.156.11",
493
+ "@version": "1",
494
+ "host": "skywalker",
495
+ "httpversion": "1.1",
496
+ "timestamp": "27/Aug/2014:02:44:04 +0000"
497
+ }
498
+ }]
499
+ }
500
+ }
data/spec/filters/integration/elasticsearch_spec.rb ADDED
@@ -0,0 +1,44 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/plugin"
4
+ require "logstash/filters/elasticsearch"
5
+
6
+ describe LogStash::Filters::Elasticsearch, :integration => true do
7
+
8
+ let(:config) do
9
+ {
10
+ "hosts" => ["localhost:9200"],
11
+ "query" => "response: 404",
12
+ "fields" => [ ["response", "code"] ],
13
+ }
14
+ end
15
+ let(:plugin) { described_class.new(config) }
16
+ let(:event) { LogStash::Event.new({}) }
17
+
18
+ before(:each) do
19
+ plugin.register
20
+ end
21
+
22
+ it "should enhance the current event with new data" do
23
+ plugin.filter(event)
24
+ expect(event["code"]).to eq(404)
25
+ end
26
+
27
+ context "when retrieving a list of elements" do
28
+
29
+ let(:config) do
30
+ {
31
+ "hosts" => ["localhost:9200"],
32
+ "query" => "response: 404",
33
+ "fields" => [ ["response", "code"] ],
34
+ "result_size" => 10
35
+ }
36
+ end
37
+
38
+ it "should enhance the current event with new data" do
39
+ plugin.filter(event)
40
+ expect(event["code"]).to eq([404]*10)
41
+ end
42
+
43
+ end
44
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.4
4
+ version: 2.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-24 00:00:00.000000000 Z
11
+ date: 2016-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,7 +52,7 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
- description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
55
+ description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
56
56
  email: info@elastic.co
57
57
  executables: []
58
58
  extensions: []
@@ -65,8 +65,12 @@ files:
65
65
  - NOTICE.TXT
66
66
  - README.md
67
67
  - lib/logstash/filters/elasticsearch.rb
68
+ - lib/logstash/filters/elasticsearch/client.rb
68
69
  - logstash-filter-elasticsearch.gemspec
69
70
  - spec/filters/elasticsearch_spec.rb
71
+ - spec/filters/fixtures/request_x_1.json
72
+ - spec/filters/fixtures/request_x_10.json
73
+ - spec/filters/integration/elasticsearch_spec.rb
70
74
  homepage: http://www.elastic.co/guide/en/logstash/current/index.html
71
75
  licenses:
72
76
  - Apache License (2.0)
@@ -95,3 +99,6 @@ specification_version: 4
95
99
  summary: Search elasticsearch for a previous log event and copy some fields from it into the current event
96
100
  test_files:
97
101
  - spec/filters/elasticsearch_spec.rb
102
+ - spec/filters/fixtures/request_x_1.json
103
+ - spec/filters/fixtures/request_x_10.json
104
+ - spec/filters/integration/elasticsearch_spec.rb