logstash-filter-elasticsearch 2.0.4 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -4
- data/Gemfile +2 -1
- data/LICENSE +1 -1
- data/README.md +12 -3
- data/lib/logstash/filters/elasticsearch.rb +39 -37
- data/lib/logstash/filters/elasticsearch/client.rb +35 -0
- data/logstash-filter-elasticsearch.gemspec +2 -2
- data/spec/filters/elasticsearch_spec.rb +66 -1
- data/spec/filters/fixtures/request_x_1.json +62 -0
- data/spec/filters/fixtures/request_x_10.json +500 -0
- data/spec/filters/integration/elasticsearch_spec.rb +44 -0
- metadata +10 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cfc8b5e9cf33d9d7ef26f24fa0692a8e64119c17
|
|
4
|
+
data.tar.gz: 84e9261001fa6a28fd111391fcb23b36c41ce834
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 853a1678509361a7eedd80a0fc8264ac7184b0aaf6358cf0373ee0a9783af96620bf873cd12a5d08ba0c9bb5f8966883d68f36fc0c228e64a01b82ed8292134f
|
|
7
|
+
data.tar.gz: bf37c8e634112d4d5b8cb86cd018cd200e6f91fa11bffabc0fec9e5f38f6dd00eb0404ae4d8fe99fc79ee68df47b5f912a91f4d594b2c54e54169ec9564eb260
|
data/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
|
-
|
|
1
|
+
## 2.1.0
|
|
2
|
+
- Improved the configuration options to be more easy to understand and
|
|
3
|
+
match what the expectations are from the documentation.
|
|
4
|
+
- Initial refactoring to include later one a common client for all the
|
|
5
|
+
ES plugins.
|
|
6
|
+
- Adding support for having an index in the query pattern.
|
|
7
|
+
- Improved documentation.
|
|
8
|
+
- Added intitial integration and unit tests.
|
|
9
|
+
## 2.0.4
|
|
2
10
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
|
3
|
-
|
|
11
|
+
## 2.0.3
|
|
4
12
|
- New dependency requirements for logstash-core for the 5.0 release
|
|
5
13
|
## 2.0.0
|
|
6
14
|
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
|
7
15
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
|
8
16
|
- Dependency on logstash-core update to 2.0
|
|
9
|
-
|
|
10
17
|
## 0.1.6
|
|
11
|
-
|
|
12
18
|
- removed require statement for a file that is no longer present in logstash-core.
|
data/Gemfile
CHANGED
data/LICENSE
CHANGED
data/README.md
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# Logstash Plugin
|
|
2
2
|
|
|
3
|
-
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-elasticsearch-unit/)
|
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-filter-elasticsearch)
|
|
5
4
|
|
|
6
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
|
7
6
|
|
|
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
|
|
56
55
|
```
|
|
57
56
|
- Install plugin
|
|
58
57
|
```sh
|
|
58
|
+
# Logstash 2.3 and higher
|
|
59
|
+
bin/logstash-plugin install --no-verify
|
|
60
|
+
|
|
61
|
+
# Prior to Logstash 2.3
|
|
59
62
|
bin/plugin install --no-verify
|
|
63
|
+
|
|
60
64
|
```
|
|
61
65
|
- Run Logstash with your plugin
|
|
62
66
|
```sh
|
|
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
|
|
|
74
78
|
```
|
|
75
79
|
- Install the plugin from the Logstash home
|
|
76
80
|
```sh
|
|
77
|
-
|
|
81
|
+
# Logstash 2.3 and higher
|
|
82
|
+
bin/logstash-plugin install --no-verify
|
|
83
|
+
|
|
84
|
+
# Prior to Logstash 2.3
|
|
85
|
+
bin/plugin install --no-verify
|
|
86
|
+
|
|
78
87
|
```
|
|
79
88
|
- Start Logstash and proceed to test the plugin
|
|
80
89
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
1
2
|
require "logstash/filters/base"
|
|
2
3
|
require "logstash/namespace"
|
|
3
|
-
|
|
4
|
+
require_relative "elasticsearch/client"
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
# Search elasticsearch for a previous log event and copy some fields from it
|
|
@@ -15,7 +16,7 @@ require "base64"
|
|
|
15
16
|
# elasticsearch {
|
|
16
17
|
# hosts => ["es-server"]
|
|
17
18
|
# query => "type:start AND operation:%{[opid]}"
|
|
18
|
-
# fields => ["@timestamp", "started"]
|
|
19
|
+
# fields => [["@timestamp", "started"]]
|
|
19
20
|
# }
|
|
20
21
|
#
|
|
21
22
|
# date {
|
|
@@ -32,9 +33,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
|
32
33
|
config_name "elasticsearch"
|
|
33
34
|
|
|
34
35
|
# List of elasticsearch hosts to use for querying.
|
|
35
|
-
config :hosts, :validate => :array
|
|
36
|
+
config :hosts, :validate => :array, :default => [ "localhost:9200" ]
|
|
37
|
+
|
|
38
|
+
# Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
|
|
39
|
+
config :index, :validate => :string, :default => ""
|
|
36
40
|
|
|
37
|
-
# Elasticsearch query string
|
|
41
|
+
# Elasticsearch query string. Read the Elasticsearch query string documentation
|
|
42
|
+
# for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
|
|
38
43
|
config :query, :validate => :string
|
|
39
44
|
|
|
40
45
|
# Comma-delimited list of `<field>:<direction>` pairs that define the sort order
|
|
@@ -55,49 +60,46 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
|
55
60
|
# SSL Certificate Authority file
|
|
56
61
|
config :ca_file, :validate => :path
|
|
57
62
|
|
|
63
|
+
# Whether results should be sorted or not
|
|
64
|
+
config :enable_sort, :validate => :boolean, :default => true
|
|
58
65
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
require "elasticsearch"
|
|
62
|
-
|
|
63
|
-
transport_options = {}
|
|
64
|
-
|
|
65
|
-
if @user && @password
|
|
66
|
-
token = Base64.strict_encode64("#{@user}:#{@password.value}")
|
|
67
|
-
transport_options[:headers] = { Authorization: "Basic #{token}" }
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
hosts = if @ssl then
|
|
71
|
-
@hosts.map {|h| { host: h, scheme: 'https' } }
|
|
72
|
-
else
|
|
73
|
-
@hosts
|
|
74
|
-
end
|
|
66
|
+
# How many results to return
|
|
67
|
+
config :result_size, :validate => :number, :default => 1
|
|
75
68
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
end
|
|
69
|
+
# Tags the event on failure to look up geo information. This can be used in later analysis.
|
|
70
|
+
config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
|
|
79
71
|
|
|
80
|
-
|
|
81
|
-
|
|
72
|
+
def register
|
|
73
|
+
options = {
|
|
74
|
+
:ssl => @ssl,
|
|
75
|
+
:hosts => @hosts,
|
|
76
|
+
:ca_file => @ca_file,
|
|
77
|
+
:logger => @logger,
|
|
78
|
+
:index => @index
|
|
79
|
+
}
|
|
80
|
+
@client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
|
|
82
81
|
end # def register
|
|
83
82
|
|
|
84
|
-
public
|
|
85
83
|
def filter(event)
|
|
86
|
-
|
|
87
|
-
|
|
88
84
|
begin
|
|
89
85
|
query_str = event.sprintf(@query)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
86
|
+
params = { :q => query_str, :size => result_size }
|
|
87
|
+
params[:sort] = @sort if @enable_sort
|
|
88
|
+
results = @client.search(params)
|
|
89
|
+
|
|
90
|
+
@fields.each do |old_key, new_key|
|
|
91
|
+
if !results['hits']['hits'].empty?
|
|
92
|
+
set = []
|
|
93
|
+
results["hits"]["hits"].to_a.each do |doc|
|
|
94
|
+
set << doc["_source"][old_key]
|
|
95
|
+
end
|
|
96
|
+
event[new_key] = ( set.count > 1 ? set : set.first)
|
|
97
|
+
end
|
|
95
98
|
end
|
|
96
|
-
|
|
97
|
-
filter_matched(event)
|
|
98
99
|
rescue => e
|
|
99
|
-
@logger.warn("Failed to query elasticsearch for previous event",
|
|
100
|
-
|
|
100
|
+
@logger.warn("Failed to query elasticsearch for previous event", :index, @index, :query => query_str, :event => event, :error => e)
|
|
101
|
+
@tag_on_failure.each{|tag| event.tag(tag)}
|
|
101
102
|
end
|
|
103
|
+
filter_matched(event)
|
|
102
104
|
end # def filter
|
|
103
105
|
end # class LogStash::Filters::Elasticsearch
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "elasticsearch"
|
|
3
|
+
require "base64"
|
|
4
|
+
|
|
5
|
+
module LogStash
|
|
6
|
+
module Filters
|
|
7
|
+
class ElasticsearchClient
|
|
8
|
+
|
|
9
|
+
attr_reader :client
|
|
10
|
+
|
|
11
|
+
def initialize(user, password, options={})
|
|
12
|
+
ssl = options.fetch(:ssh, false)
|
|
13
|
+
hosts = options[:hosts]
|
|
14
|
+
@logger = options[:logger]
|
|
15
|
+
|
|
16
|
+
transport_options = {}
|
|
17
|
+
if user && password
|
|
18
|
+
token = ::Base64.strict_encode64("#{user}:#{password.value}")
|
|
19
|
+
transport_options[:headers] = { Authorization: "Basic #{token}" }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
host.map! {|h| { host: h, scheme: 'https' } } if ssl
|
|
23
|
+
transport_options[:ssl] = { ca_file: options[:ca_file] } if ssl && options[:ca_file]
|
|
24
|
+
|
|
25
|
+
@logger.info("New ElasticSearch filter", :hosts => hosts)
|
|
26
|
+
@client = ::Elasticsearch::Client.new(index: options[:index], hosts: hosts, transport_options: transport_options)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def search(params)
|
|
30
|
+
@client.search(params)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
|
4
|
-
s.version = '2.0
|
|
4
|
+
s.version = '2.1.0'
|
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
|
6
6
|
s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
|
|
7
|
-
s.description = "This gem is a
|
|
7
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
8
8
|
s.authors = ["Elastic"]
|
|
9
9
|
s.email = 'info@elastic.co'
|
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# encoding: utf-8
|
|
2
|
-
|
|
3
2
|
require "logstash/devutils/rspec/spec_helper"
|
|
4
3
|
require "logstash/plugin"
|
|
5
4
|
require "logstash/filters/elasticsearch"
|
|
5
|
+
require "logstash/json"
|
|
6
6
|
|
|
7
7
|
describe LogStash::Filters::Elasticsearch do
|
|
8
8
|
|
|
@@ -15,4 +15,69 @@ describe LogStash::Filters::Elasticsearch do
|
|
|
15
15
|
end
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
describe "data fetch" do
|
|
19
|
+
let(:config) do
|
|
20
|
+
{
|
|
21
|
+
"hosts" => ["localhost:9200"],
|
|
22
|
+
"query" => "response: 404",
|
|
23
|
+
"fields" => [ ["response", "code"] ],
|
|
24
|
+
}
|
|
25
|
+
end
|
|
26
|
+
let(:plugin) { described_class.new(config) }
|
|
27
|
+
let(:event) { LogStash::Event.new({}) }
|
|
28
|
+
|
|
29
|
+
let(:response) do
|
|
30
|
+
LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
let(:client) { double(:client) }
|
|
34
|
+
|
|
35
|
+
before(:each) do
|
|
36
|
+
allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
|
|
37
|
+
allow(client).to receive(:search).and_return(response)
|
|
38
|
+
plugin.register
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
it "should enhance the current event with new data" do
|
|
42
|
+
plugin.filter(event)
|
|
43
|
+
expect(event["code"]).to eq(404)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
context "when asking for more than one result" do
|
|
47
|
+
|
|
48
|
+
let(:config) do
|
|
49
|
+
{
|
|
50
|
+
"hosts" => ["localhost:9200"],
|
|
51
|
+
"query" => "response: 404",
|
|
52
|
+
"fields" => [ ["response", "code"] ],
|
|
53
|
+
"result_size" => 10
|
|
54
|
+
}
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
let(:response) do
|
|
58
|
+
LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
it "should enhance the current event with new data" do
|
|
62
|
+
plugin.filter(event)
|
|
63
|
+
expect(event["code"]).to eq([404]*10)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
context "if something wrong happen during connection" do
|
|
68
|
+
|
|
69
|
+
before(:each) do
|
|
70
|
+
allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
|
|
71
|
+
allow(client).to receive(:search).and_raise("connection exception")
|
|
72
|
+
plugin.register
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
it "tag the event as something happened, but still deliver it" do
|
|
76
|
+
expect(plugin.logger).to receive(:warn)
|
|
77
|
+
plugin.filter(event)
|
|
78
|
+
expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
18
83
|
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"took": 49,
|
|
3
|
+
"timed_out": false,
|
|
4
|
+
"_shards": {
|
|
5
|
+
"total": 155,
|
|
6
|
+
"successful": 155,
|
|
7
|
+
"failed": 0
|
|
8
|
+
},
|
|
9
|
+
"hits": {
|
|
10
|
+
"total": 13476,
|
|
11
|
+
"max_score": 1,
|
|
12
|
+
"hits": [{
|
|
13
|
+
"_index": "logstash-2014.08.26",
|
|
14
|
+
"_type": "logs",
|
|
15
|
+
"_id": "AVVY76L_AW7v0kX8KXo4",
|
|
16
|
+
"_score": 1,
|
|
17
|
+
"_source": {
|
|
18
|
+
"request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
|
|
19
|
+
"agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
20
|
+
"geoip": {
|
|
21
|
+
"timezone": "America/Los_Angeles",
|
|
22
|
+
"ip": "66.249.73.185",
|
|
23
|
+
"latitude": 37.386,
|
|
24
|
+
"continent_code": "NA",
|
|
25
|
+
"city_name": "Mountain View",
|
|
26
|
+
"country_code2": "US",
|
|
27
|
+
"country_name": "United States",
|
|
28
|
+
"dma_code": 807,
|
|
29
|
+
"country_code3": "US",
|
|
30
|
+
"region_name": "California",
|
|
31
|
+
"location": [-122.0838,
|
|
32
|
+
37.386
|
|
33
|
+
],
|
|
34
|
+
"postal_code": "94035",
|
|
35
|
+
"longitude": -122.0838,
|
|
36
|
+
"region_code": "CA"
|
|
37
|
+
},
|
|
38
|
+
"auth": "-",
|
|
39
|
+
"ident": "-",
|
|
40
|
+
"verb": "GET",
|
|
41
|
+
"useragent": {
|
|
42
|
+
"os": "Other",
|
|
43
|
+
"major": "2",
|
|
44
|
+
"minor": "1",
|
|
45
|
+
"name": "Googlebot",
|
|
46
|
+
"os_name": "Other",
|
|
47
|
+
"device": "Spider"
|
|
48
|
+
},
|
|
49
|
+
"message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
50
|
+
"referrer": "\"-\"",
|
|
51
|
+
"@timestamp": "2014-08-26T21:22:13.000Z",
|
|
52
|
+
"response": 404,
|
|
53
|
+
"bytes": 294,
|
|
54
|
+
"clientip": "66.249.73.185",
|
|
55
|
+
"@version": "1",
|
|
56
|
+
"host": "skywalker",
|
|
57
|
+
"httpversion": "1.1",
|
|
58
|
+
"timestamp": "26/Aug/2014:21:22:13 +0000"
|
|
59
|
+
}
|
|
60
|
+
}]
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
{
|
|
2
|
+
"took": 49,
|
|
3
|
+
"timed_out": false,
|
|
4
|
+
"_shards": {
|
|
5
|
+
"total": 155,
|
|
6
|
+
"successful": 155,
|
|
7
|
+
"failed": 0
|
|
8
|
+
},
|
|
9
|
+
"hits": {
|
|
10
|
+
"total": 13476,
|
|
11
|
+
"max_score": 1,
|
|
12
|
+
"hits": [{
|
|
13
|
+
"_index": "logstash-2014.08.26",
|
|
14
|
+
"_type": "logs",
|
|
15
|
+
"_id": "AVVY76L_AW7v0kX8KXo4",
|
|
16
|
+
"_score": 1,
|
|
17
|
+
"_source": {
|
|
18
|
+
"request": "/doc/index.html?org/elasticsearch/action/search/SearchResponse.html",
|
|
19
|
+
"agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
20
|
+
"geoip": {
|
|
21
|
+
"timezone": "America/Los_Angeles",
|
|
22
|
+
"ip": "66.249.73.185",
|
|
23
|
+
"latitude": 37.386,
|
|
24
|
+
"continent_code": "NA",
|
|
25
|
+
"city_name": "Mountain View",
|
|
26
|
+
"country_code2": "US",
|
|
27
|
+
"country_name": "United States",
|
|
28
|
+
"dma_code": 807,
|
|
29
|
+
"country_code3": "US",
|
|
30
|
+
"region_name": "California",
|
|
31
|
+
"location": [-122.0838,
|
|
32
|
+
37.386
|
|
33
|
+
],
|
|
34
|
+
"postal_code": "94035",
|
|
35
|
+
"longitude": -122.0838,
|
|
36
|
+
"region_code": "CA"
|
|
37
|
+
},
|
|
38
|
+
"auth": "-",
|
|
39
|
+
"ident": "-",
|
|
40
|
+
"verb": "GET",
|
|
41
|
+
"useragent": {
|
|
42
|
+
"os": "Other",
|
|
43
|
+
"major": "2",
|
|
44
|
+
"minor": "1",
|
|
45
|
+
"name": "Googlebot",
|
|
46
|
+
"os_name": "Other",
|
|
47
|
+
"device": "Spider"
|
|
48
|
+
},
|
|
49
|
+
"message": "66.249.73.185 - - [26/Aug/2014:21:22:13 +0000] \"GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1\" 404 294 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
50
|
+
"referrer": "\"-\"",
|
|
51
|
+
"@timestamp": "2014-08-26T21:22:13.000Z",
|
|
52
|
+
"response": 404,
|
|
53
|
+
"bytes": 294,
|
|
54
|
+
"clientip": "66.249.73.185",
|
|
55
|
+
"@version": "1",
|
|
56
|
+
"host": "skywalker",
|
|
57
|
+
"httpversion": "1.1",
|
|
58
|
+
"timestamp": "26/Aug/2014:21:22:13 +0000"
|
|
59
|
+
}
|
|
60
|
+
}, {
|
|
61
|
+
"_index": "logstash-2014.08.26",
|
|
62
|
+
"_type": "logs",
|
|
63
|
+
"_id": "AVVY76eJAW7v0kX8KXtH",
|
|
64
|
+
"_score": 1,
|
|
65
|
+
"_source": {
|
|
66
|
+
"request": "/presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif",
|
|
67
|
+
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1\"",
|
|
68
|
+
"geoip": {
|
|
69
|
+
"timezone": "Asia/Shanghai",
|
|
70
|
+
"ip": "111.199.235.239",
|
|
71
|
+
"latitude": 39.9289,
|
|
72
|
+
"continent_code": "AS",
|
|
73
|
+
"city_name": "Beijing",
|
|
74
|
+
"country_code2": "CN",
|
|
75
|
+
"country_name": "China",
|
|
76
|
+
"dma_code": null,
|
|
77
|
+
"country_code3": "CN",
|
|
78
|
+
"region_name": "Beijing",
|
|
79
|
+
"location": [
|
|
80
|
+
116.3883,
|
|
81
|
+
39.9289
|
|
82
|
+
],
|
|
83
|
+
"postal_code": null,
|
|
84
|
+
"longitude": 116.3883,
|
|
85
|
+
"region_code": "11"
|
|
86
|
+
},
|
|
87
|
+
"auth": "-",
|
|
88
|
+
"ident": "-",
|
|
89
|
+
"verb": "GET",
|
|
90
|
+
"useragent": {
|
|
91
|
+
"patch": "5",
|
|
92
|
+
"os": "Mac OS X 10.8.5",
|
|
93
|
+
"major": "6",
|
|
94
|
+
"minor": "0",
|
|
95
|
+
"os_minor": "8",
|
|
96
|
+
"os_major": "10",
|
|
97
|
+
"name": "Safari",
|
|
98
|
+
"os_name": "Mac OS X",
|
|
99
|
+
"device": "Other"
|
|
100
|
+
},
|
|
101
|
+
"message": "111.199.235.239 - - [26/Aug/2014:22:06:06 +0000] \"GET /presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif HTTP/1.1\" 404 364 \"http://semicomplete.com/presentations/logstash-puppetconf-2012/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/536.30.1 (KHTML, like Gecko) Version/6.0.5 Safari/536.30.1\"",
|
|
102
|
+
"referrer": "\"http://semicomplete.com/presentations/logstash-puppetconf-2012/\"",
|
|
103
|
+
"@timestamp": "2014-08-26T22:06:06.000Z",
|
|
104
|
+
"response": 404,
|
|
105
|
+
"bytes": 364,
|
|
106
|
+
"clientip": "111.199.235.239",
|
|
107
|
+
"@version": "1",
|
|
108
|
+
"host": "skywalker",
|
|
109
|
+
"httpversion": "1.1",
|
|
110
|
+
"timestamp": "26/Aug/2014:22:06:06 +0000"
|
|
111
|
+
}
|
|
112
|
+
}, {
|
|
113
|
+
"_index": "logstash-2014.08.26",
|
|
114
|
+
"_type": "logs",
|
|
115
|
+
"_id": "AVVY76eJAW7v0kX8KXtf",
|
|
116
|
+
"_score": 1,
|
|
117
|
+
"_source": {
|
|
118
|
+
"request": "/files/logstash/logstash-1.3.2-monolithic.jar",
|
|
119
|
+
"agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
120
|
+
"geoip": {
|
|
121
|
+
"timezone": "America/Los_Angeles",
|
|
122
|
+
"ip": "208.91.156.11",
|
|
123
|
+
"latitude": 34.0486,
|
|
124
|
+
"continent_code": "NA",
|
|
125
|
+
"city_name": "Los Angeles",
|
|
126
|
+
"country_code2": "US",
|
|
127
|
+
"country_name": "United States",
|
|
128
|
+
"dma_code": 803,
|
|
129
|
+
"country_code3": "US",
|
|
130
|
+
"region_name": "California",
|
|
131
|
+
"location": [-118.4424,
|
|
132
|
+
34.0486
|
|
133
|
+
],
|
|
134
|
+
"postal_code": "90025",
|
|
135
|
+
"longitude": -118.4424,
|
|
136
|
+
"region_code": "CA"
|
|
137
|
+
},
|
|
138
|
+
"auth": "-",
|
|
139
|
+
"ident": "-",
|
|
140
|
+
"verb": "GET",
|
|
141
|
+
"useragent": {
|
|
142
|
+
"os": "Other",
|
|
143
|
+
"name": "Other",
|
|
144
|
+
"os_name": "Other",
|
|
145
|
+
"device": "Other"
|
|
146
|
+
},
|
|
147
|
+
"message": "208.91.156.11 - - [26/Aug/2014:22:12:14 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
148
|
+
"referrer": "\"-\"",
|
|
149
|
+
"@timestamp": "2014-08-26T22:12:14.000Z",
|
|
150
|
+
"response": 404,
|
|
151
|
+
"bytes": 324,
|
|
152
|
+
"clientip": "208.91.156.11",
|
|
153
|
+
"@version": "1",
|
|
154
|
+
"host": "skywalker",
|
|
155
|
+
"httpversion": "1.1",
|
|
156
|
+
"timestamp": "26/Aug/2014:22:12:14 +0000"
|
|
157
|
+
}
|
|
158
|
+
}, {
|
|
159
|
+
"_index": "logstash-2014.08.26",
|
|
160
|
+
"_type": "logs",
|
|
161
|
+
"_id": "AVVY761xAW7v0kX8KXvw",
|
|
162
|
+
"_score": 1,
|
|
163
|
+
"_source": {
|
|
164
|
+
"request": "/files/logstash/logstash-1.3.2-monolithic.jar",
|
|
165
|
+
"agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
166
|
+
"geoip": {
|
|
167
|
+
"timezone": "America/Los_Angeles",
|
|
168
|
+
"ip": "208.91.156.11",
|
|
169
|
+
"latitude": 34.0486,
|
|
170
|
+
"continent_code": "NA",
|
|
171
|
+
"city_name": "Los Angeles",
|
|
172
|
+
"country_code2": "US",
|
|
173
|
+
"country_name": "United States",
|
|
174
|
+
"dma_code": 803,
|
|
175
|
+
"country_code3": "US",
|
|
176
|
+
"region_name": "California",
|
|
177
|
+
"location": [-118.4424,
|
|
178
|
+
34.0486
|
|
179
|
+
],
|
|
180
|
+
"postal_code": "90025",
|
|
181
|
+
"longitude": -118.4424,
|
|
182
|
+
"region_code": "CA"
|
|
183
|
+
},
|
|
184
|
+
"auth": "-",
|
|
185
|
+
"ident": "-",
|
|
186
|
+
"verb": "GET",
|
|
187
|
+
"useragent": {
|
|
188
|
+
"os": "Other",
|
|
189
|
+
"name": "Other",
|
|
190
|
+
"os_name": "Other",
|
|
191
|
+
"device": "Other"
|
|
192
|
+
},
|
|
193
|
+
"message": "208.91.156.11 - - [26/Aug/2014:22:42:22 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
194
|
+
"referrer": "\"-\"",
|
|
195
|
+
"@timestamp": "2014-08-26T22:42:22.000Z",
|
|
196
|
+
"response": 404,
|
|
197
|
+
"bytes": 324,
|
|
198
|
+
"clientip": "208.91.156.11",
|
|
199
|
+
"@version": "1",
|
|
200
|
+
"host": "skywalker",
|
|
201
|
+
"httpversion": "1.1",
|
|
202
|
+
"timestamp": "26/Aug/2014:22:42:22 +0000"
|
|
203
|
+
}
|
|
204
|
+
}, {
|
|
205
|
+
"_index": "logstash-2014.08.26",
|
|
206
|
+
"_type": "logs",
|
|
207
|
+
"_id": "AVVY77AwAW7v0kX8KXx8",
|
|
208
|
+
"_score": 1,
|
|
209
|
+
"_source": {
|
|
210
|
+
"request": "/wp-login.php",
|
|
211
|
+
"agent": "\"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13\"",
|
|
212
|
+
"geoip": {
|
|
213
|
+
"timezone": "Europe/Rome",
|
|
214
|
+
"ip": "195.250.34.144",
|
|
215
|
+
"latitude": 43.4995,
|
|
216
|
+
"continent_code": "EU",
|
|
217
|
+
"city_name": "Arezzo",
|
|
218
|
+
"country_code2": "IT",
|
|
219
|
+
"country_name": "Italy",
|
|
220
|
+
"dma_code": null,
|
|
221
|
+
"country_code3": "IT",
|
|
222
|
+
"region_name": "Province of Arezzo",
|
|
223
|
+
"location": [
|
|
224
|
+
11.9109,
|
|
225
|
+
43.4995
|
|
226
|
+
],
|
|
227
|
+
"postal_code": "52100",
|
|
228
|
+
"longitude": 11.9109,
|
|
229
|
+
"region_code": "AR"
|
|
230
|
+
},
|
|
231
|
+
"auth": "-",
|
|
232
|
+
"ident": "-",
|
|
233
|
+
"verb": "GET",
|
|
234
|
+
"useragent": {
|
|
235
|
+
"patch": "1290",
|
|
236
|
+
"os": "Windows 7",
|
|
237
|
+
"major": "24",
|
|
238
|
+
"minor": "0",
|
|
239
|
+
"name": "Chrome",
|
|
240
|
+
"os_name": "Windows 7",
|
|
241
|
+
"device": "Other"
|
|
242
|
+
},
|
|
243
|
+
"message": "195.250.34.144 - - [26/Aug/2014:23:40:50 +0000] \"GET /wp-login.php HTTP/1.1\" 404 292 \"-\" \"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13\"",
|
|
244
|
+
"referrer": "\"-\"",
|
|
245
|
+
"@timestamp": "2014-08-26T23:40:50.000Z",
|
|
246
|
+
"response": 404,
|
|
247
|
+
"bytes": 292,
|
|
248
|
+
"clientip": "195.250.34.144",
|
|
249
|
+
"@version": "1",
|
|
250
|
+
"host": "skywalker",
|
|
251
|
+
"httpversion": "1.1",
|
|
252
|
+
"timestamp": "26/Aug/2014:23:40:50 +0000"
|
|
253
|
+
}
|
|
254
|
+
}, {
|
|
255
|
+
"_index": "logstash-2014.08.26",
|
|
256
|
+
"_type": "logs",
|
|
257
|
+
"_id": "AVVY77AwAW7v0kX8KXyB",
|
|
258
|
+
"_score": 1,
|
|
259
|
+
"_source": {
|
|
260
|
+
"request": "/presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif",
|
|
261
|
+
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/6.1.1 Safari/537.73.11\"",
|
|
262
|
+
"geoip": {
|
|
263
|
+
"timezone": "Asia/Kolkata",
|
|
264
|
+
"ip": "122.166.142.108",
|
|
265
|
+
"latitude": 12.9833,
|
|
266
|
+
"continent_code": "AS",
|
|
267
|
+
"city_name": "Bengaluru",
|
|
268
|
+
"country_code2": "IN",
|
|
269
|
+
"country_name": "India",
|
|
270
|
+
"dma_code": null,
|
|
271
|
+
"country_code3": "IN",
|
|
272
|
+
"region_name": "Karnataka",
|
|
273
|
+
"location": [
|
|
274
|
+
77.5833,
|
|
275
|
+
12.9833
|
|
276
|
+
],
|
|
277
|
+
"postal_code": null,
|
|
278
|
+
"longitude": 77.5833,
|
|
279
|
+
"region_code": "KA"
|
|
280
|
+
},
|
|
281
|
+
"auth": "-",
|
|
282
|
+
"ident": "-",
|
|
283
|
+
"verb": "GET",
|
|
284
|
+
"useragent": {
|
|
285
|
+
"patch": "1",
|
|
286
|
+
"os": "Mac OS X 10.8.5",
|
|
287
|
+
"major": "6",
|
|
288
|
+
"minor": "1",
|
|
289
|
+
"os_minor": "8",
|
|
290
|
+
"os_major": "10",
|
|
291
|
+
"name": "Safari",
|
|
292
|
+
"os_name": "Mac OS X",
|
|
293
|
+
"device": "Other"
|
|
294
|
+
},
|
|
295
|
+
"message": "122.166.142.108 - - [26/Aug/2014:23:41:19 +0000] \"GET /presentations/logstash-puppetconf-2012/images/office-space-printer-beat-down-gif.gif HTTP/1.1\" 404 364 \"http://semicomplete.com/presentations/logstash-puppetconf-2012/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/6.1.1 Safari/537.73.11\"",
|
|
296
|
+
"referrer": "\"http://semicomplete.com/presentations/logstash-puppetconf-2012/\"",
|
|
297
|
+
"@timestamp": "2014-08-26T23:41:19.000Z",
|
|
298
|
+
"response": 404,
|
|
299
|
+
"bytes": 364,
|
|
300
|
+
"clientip": "122.166.142.108",
|
|
301
|
+
"@version": "1",
|
|
302
|
+
"host": "skywalker",
|
|
303
|
+
"httpversion": "1.1",
|
|
304
|
+
"timestamp": "26/Aug/2014:23:41:19 +0000"
|
|
305
|
+
}
|
|
306
|
+
}, {
|
|
307
|
+
"_index": "logstash-2014.08.26",
|
|
308
|
+
"_type": "logs",
|
|
309
|
+
"_id": "AVVY77NUAW7v0kX8KX0s",
|
|
310
|
+
"_score": 1,
|
|
311
|
+
"_source": {
|
|
312
|
+
"request": "/projects/xdotool%3E",
|
|
313
|
+
"agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
314
|
+
"geoip": {
|
|
315
|
+
"timezone": "America/Los_Angeles",
|
|
316
|
+
"ip": "66.249.73.135",
|
|
317
|
+
"latitude": 37.386,
|
|
318
|
+
"continent_code": "NA",
|
|
319
|
+
"city_name": "Mountain View",
|
|
320
|
+
"country_code2": "US",
|
|
321
|
+
"country_name": "United States",
|
|
322
|
+
"dma_code": 807,
|
|
323
|
+
"country_code3": "US",
|
|
324
|
+
"region_name": "California",
|
|
325
|
+
"location": [-122.0838,
|
|
326
|
+
37.386
|
|
327
|
+
],
|
|
328
|
+
"postal_code": "94035",
|
|
329
|
+
"longitude": -122.0838,
|
|
330
|
+
"region_code": "CA"
|
|
331
|
+
},
|
|
332
|
+
"auth": "-",
|
|
333
|
+
"ident": "-",
|
|
334
|
+
"verb": "GET",
|
|
335
|
+
"useragent": {
|
|
336
|
+
"os": "Other",
|
|
337
|
+
"major": "2",
|
|
338
|
+
"minor": "1",
|
|
339
|
+
"name": "Googlebot",
|
|
340
|
+
"os_name": "Other",
|
|
341
|
+
"device": "Spider"
|
|
342
|
+
},
|
|
343
|
+
"message": "66.249.73.135 - - [26/Aug/2014:23:25:32 +0000] \"GET /projects/xdotool%3E HTTP/1.1\" 404 7861 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
344
|
+
"referrer": "\"-\"",
|
|
345
|
+
"@timestamp": "2014-08-26T23:25:32.000Z",
|
|
346
|
+
"response": 404,
|
|
347
|
+
"bytes": 7861,
|
|
348
|
+
"clientip": "66.249.73.135",
|
|
349
|
+
"@version": "1",
|
|
350
|
+
"host": "skywalker",
|
|
351
|
+
"httpversion": "1.1",
|
|
352
|
+
"timestamp": "26/Aug/2014:23:25:32 +0000"
|
|
353
|
+
}
|
|
354
|
+
}, {
|
|
355
|
+
"_index": "logstash-2014.08.27",
|
|
356
|
+
"_type": "logs",
|
|
357
|
+
"_id": "AVVY77vzAW7v0kX8KX5_",
|
|
358
|
+
"_score": 1,
|
|
359
|
+
"_source": {
|
|
360
|
+
"request": "/wp-login.php?action=register",
|
|
361
|
+
"agent": "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0\"",
|
|
362
|
+
"geoip": {
|
|
363
|
+
"timezone": "America/Chicago",
|
|
364
|
+
"ip": "198.143.145.210",
|
|
365
|
+
"latitude": 41.8825,
|
|
366
|
+
"continent_code": "NA",
|
|
367
|
+
"city_name": "Chicago",
|
|
368
|
+
"country_code2": "US",
|
|
369
|
+
"country_name": "United States",
|
|
370
|
+
"dma_code": 602,
|
|
371
|
+
"country_code3": "US",
|
|
372
|
+
"region_name": "Illinois",
|
|
373
|
+
"location": [-87.6441,
|
|
374
|
+
41.8825
|
|
375
|
+
],
|
|
376
|
+
"postal_code": "60661",
|
|
377
|
+
"longitude": -87.6441,
|
|
378
|
+
"region_code": "IL"
|
|
379
|
+
},
|
|
380
|
+
"auth": "-",
|
|
381
|
+
"ident": "-",
|
|
382
|
+
"verb": "GET",
|
|
383
|
+
"useragent": {
|
|
384
|
+
"os": "Mac OS X 10.7",
|
|
385
|
+
"major": "21",
|
|
386
|
+
"minor": "0",
|
|
387
|
+
"os_minor": "7",
|
|
388
|
+
"os_major": "10",
|
|
389
|
+
"name": "Firefox",
|
|
390
|
+
"os_name": "Mac OS X",
|
|
391
|
+
"device": "Other"
|
|
392
|
+
},
|
|
393
|
+
"message": "198.143.145.210 - - [27/Aug/2014:01:30:10 +0000] \"GET /wp-login.php?action=register HTTP/1.0\" 404 296 \"http://www.semicomplete.com/misc/sample.log\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:21.0) Gecko/20100101 Firefox/21.0\"",
|
|
394
|
+
"referrer": "\"http://www.semicomplete.com/misc/sample.log\"",
|
|
395
|
+
"@timestamp": "2014-08-27T01:30:10.000Z",
|
|
396
|
+
"response": 404,
|
|
397
|
+
"bytes": 296,
|
|
398
|
+
"clientip": "198.143.145.210",
|
|
399
|
+
"@version": "1",
|
|
400
|
+
"host": "skywalker",
|
|
401
|
+
"httpversion": "1.0",
|
|
402
|
+
"timestamp": "27/Aug/2014:01:30:10 +0000"
|
|
403
|
+
}
|
|
404
|
+
}, {
|
|
405
|
+
"_index": "logstash-2014.08.27",
|
|
406
|
+
"_type": "logs",
|
|
407
|
+
"_id": "AVVY77vzAW7v0kX8KX6w",
|
|
408
|
+
"_score": 1,
|
|
409
|
+
"_source": {
|
|
410
|
+
"request": "/projects/securitrack/config.xsl",
|
|
411
|
+
"agent": "\"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
412
|
+
"geoip": {
|
|
413
|
+
"timezone": "America/Los_Angeles",
|
|
414
|
+
"ip": "66.249.73.135",
|
|
415
|
+
"latitude": 37.386,
|
|
416
|
+
"continent_code": "NA",
|
|
417
|
+
"city_name": "Mountain View",
|
|
418
|
+
"country_code2": "US",
|
|
419
|
+
"country_name": "United States",
|
|
420
|
+
"dma_code": 807,
|
|
421
|
+
"country_code3": "US",
|
|
422
|
+
"region_name": "California",
|
|
423
|
+
"location": [-122.0838,
|
|
424
|
+
37.386
|
|
425
|
+
],
|
|
426
|
+
"postal_code": "94035",
|
|
427
|
+
"longitude": -122.0838,
|
|
428
|
+
"region_code": "CA"
|
|
429
|
+
},
|
|
430
|
+
"auth": "-",
|
|
431
|
+
"ident": "-",
|
|
432
|
+
"verb": "GET",
|
|
433
|
+
"useragent": {
|
|
434
|
+
"os": "Other",
|
|
435
|
+
"major": "2",
|
|
436
|
+
"minor": "1",
|
|
437
|
+
"name": "Googlebot",
|
|
438
|
+
"os_name": "Other",
|
|
439
|
+
"device": "Spider"
|
|
440
|
+
},
|
|
441
|
+
"message": "66.249.73.135 - - [27/Aug/2014:01:40:51 +0000] \"GET /projects/securitrack/config.xsl HTTP/1.1\" 404 315 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"",
|
|
442
|
+
"referrer": "\"-\"",
|
|
443
|
+
"@timestamp": "2014-08-27T01:40:51.000Z",
|
|
444
|
+
"response": 404,
|
|
445
|
+
"bytes": 315,
|
|
446
|
+
"clientip": "66.249.73.135",
|
|
447
|
+
"@version": "1",
|
|
448
|
+
"host": "skywalker",
|
|
449
|
+
"httpversion": "1.1",
|
|
450
|
+
"timestamp": "27/Aug/2014:01:40:51 +0000"
|
|
451
|
+
}
|
|
452
|
+
}, {
|
|
453
|
+
"_index": "logstash-2014.08.27",
|
|
454
|
+
"_type": "logs",
|
|
455
|
+
"_id": "AVVY78FiAW7v0kX8KYBM",
|
|
456
|
+
"_score": 1,
|
|
457
|
+
"_source": {
|
|
458
|
+
"request": "/files/logstash/logstash-1.3.2-monolithic.jar",
|
|
459
|
+
"agent": "\"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
460
|
+
"geoip": {
|
|
461
|
+
"timezone": "America/Los_Angeles",
|
|
462
|
+
"ip": "208.91.156.11",
|
|
463
|
+
"latitude": 34.0486,
|
|
464
|
+
"continent_code": "NA",
|
|
465
|
+
"city_name": "Los Angeles",
|
|
466
|
+
"country_code2": "US",
|
|
467
|
+
"country_name": "United States",
|
|
468
|
+
"dma_code": 803,
|
|
469
|
+
"country_code3": "US",
|
|
470
|
+
"region_name": "California",
|
|
471
|
+
"location": [-118.4424,
|
|
472
|
+
34.0486
|
|
473
|
+
],
|
|
474
|
+
"postal_code": "90025",
|
|
475
|
+
"longitude": -118.4424,
|
|
476
|
+
"region_code": "CA"
|
|
477
|
+
},
|
|
478
|
+
"auth": "-",
|
|
479
|
+
"ident": "-",
|
|
480
|
+
"verb": "GET",
|
|
481
|
+
"useragent": {
|
|
482
|
+
"os": "Other",
|
|
483
|
+
"name": "Other",
|
|
484
|
+
"os_name": "Other",
|
|
485
|
+
"device": "Other"
|
|
486
|
+
},
|
|
487
|
+
"message": "208.91.156.11 - - [27/Aug/2014:02:44:04 +0000] \"GET /files/logstash/logstash-1.3.2-monolithic.jar HTTP/1.1\" 404 324 \"-\" \"Chef Client/10.18.2 (ruby-1.9.3-p327; ohai-6.16.0; x86_64-linux; +http://opscode.com)\"",
|
|
488
|
+
"referrer": "\"-\"",
|
|
489
|
+
"@timestamp": "2014-08-27T02:44:04.000Z",
|
|
490
|
+
"response": 404,
|
|
491
|
+
"bytes": 324,
|
|
492
|
+
"clientip": "208.91.156.11",
|
|
493
|
+
"@version": "1",
|
|
494
|
+
"host": "skywalker",
|
|
495
|
+
"httpversion": "1.1",
|
|
496
|
+
"timestamp": "27/Aug/2014:02:44:04 +0000"
|
|
497
|
+
}
|
|
498
|
+
}]
|
|
499
|
+
}
|
|
500
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
|
3
|
+
require "logstash/plugin"
|
|
4
|
+
require "logstash/filters/elasticsearch"
|
|
5
|
+
|
|
6
|
+
describe LogStash::Filters::Elasticsearch, :integration => true do
|
|
7
|
+
|
|
8
|
+
let(:config) do
|
|
9
|
+
{
|
|
10
|
+
"hosts" => ["localhost:9200"],
|
|
11
|
+
"query" => "response: 404",
|
|
12
|
+
"fields" => [ ["response", "code"] ],
|
|
13
|
+
}
|
|
14
|
+
end
|
|
15
|
+
let(:plugin) { described_class.new(config) }
|
|
16
|
+
let(:event) { LogStash::Event.new({}) }
|
|
17
|
+
|
|
18
|
+
before(:each) do
|
|
19
|
+
plugin.register
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
it "should enhance the current event with new data" do
|
|
23
|
+
plugin.filter(event)
|
|
24
|
+
expect(event["code"]).to eq(404)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
context "when retrieving a list of elements" do
|
|
28
|
+
|
|
29
|
+
let(:config) do
|
|
30
|
+
{
|
|
31
|
+
"hosts" => ["localhost:9200"],
|
|
32
|
+
"query" => "response: 404",
|
|
33
|
+
"fields" => [ ["response", "code"] ],
|
|
34
|
+
"result_size" => 10
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it "should enhance the current event with new data" do
|
|
39
|
+
plugin.filter(event)
|
|
40
|
+
expect(event["code"]).to eq([404]*10)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
end
|
|
44
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-elasticsearch
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-06-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -52,7 +52,7 @@ dependencies:
|
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: '0'
|
|
55
|
-
description: This gem is a
|
|
55
|
+
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
|
56
56
|
email: info@elastic.co
|
|
57
57
|
executables: []
|
|
58
58
|
extensions: []
|
|
@@ -65,8 +65,12 @@ files:
|
|
|
65
65
|
- NOTICE.TXT
|
|
66
66
|
- README.md
|
|
67
67
|
- lib/logstash/filters/elasticsearch.rb
|
|
68
|
+
- lib/logstash/filters/elasticsearch/client.rb
|
|
68
69
|
- logstash-filter-elasticsearch.gemspec
|
|
69
70
|
- spec/filters/elasticsearch_spec.rb
|
|
71
|
+
- spec/filters/fixtures/request_x_1.json
|
|
72
|
+
- spec/filters/fixtures/request_x_10.json
|
|
73
|
+
- spec/filters/integration/elasticsearch_spec.rb
|
|
70
74
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
|
71
75
|
licenses:
|
|
72
76
|
- Apache License (2.0)
|
|
@@ -95,3 +99,6 @@ specification_version: 4
|
|
|
95
99
|
summary: Search elasticsearch for a previous log event and copy some fields from it into the current event
|
|
96
100
|
test_files:
|
|
97
101
|
- spec/filters/elasticsearch_spec.rb
|
|
102
|
+
- spec/filters/fixtures/request_x_1.json
|
|
103
|
+
- spec/filters/fixtures/request_x_10.json
|
|
104
|
+
- spec/filters/integration/elasticsearch_spec.rb
|