logstash-lite 0.2.20110122143801 → 0.2.20110203130400
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/logstash/event.rb +39 -1
- data/lib/logstash/filters/grep.rb +9 -5
- data/lib/logstash/filters/multiline.rb +9 -1
- data/lib/logstash/outputs/elasticsearch.rb +2 -0
- data/lib/logstash/web/public/js/logstash.js +23 -18
- data/lib/logstash/web/server.rb +19 -7
- data/patterns/grok-patterns +8 -0
- metadata +4 -5
- data/etc/foo.yaml +0 -6
data/lib/logstash/event.rb
CHANGED
@@ -70,7 +70,20 @@ class LogStash::Event
|
|
70
70
|
|
71
71
|
# field-related access
|
72
72
|
public
|
73
|
-
def [](key)
|
73
|
+
def [](key)
|
74
|
+
# If the key isn't in fields and it starts with an "@" sign, get it out of data instead of fields
|
75
|
+
if ! @data["@fields"].has_key?(key) and key.slice(0,1) == "@"
|
76
|
+
return @data[key]
|
77
|
+
# Exists in @fields (returns value) or doesn't start with "@" (return null)
|
78
|
+
else
|
79
|
+
return @data["@fields"][key]
|
80
|
+
end
|
81
|
+
end # def []
|
82
|
+
|
83
|
+
# TODO(sissel): the semantics of [] and []= are now different in that
|
84
|
+
# []= only allows you to assign to only fields (not metadata), but
|
85
|
+
# [] allows you to read fields and metadata.
|
86
|
+
# We should fix this. Metadata is really a namespace issue, anyway.
|
74
87
|
def []=(key, value); @data["@fields"][key] = value end # def []=
|
75
88
|
def fields; return @data["@fields"] end # def fields
|
76
89
|
|
@@ -103,4 +116,29 @@ class LogStash::Event
|
|
103
116
|
end
|
104
117
|
end # event.fields.each
|
105
118
|
end # def append
|
119
|
+
|
120
|
+
# sprintf. This could use a better method name.
|
121
|
+
# The idea is to take an event and convert it to a string based on
|
122
|
+
# any format values, delimited by ${foo} where 'foo' is a field or
|
123
|
+
# metadata member.
|
124
|
+
#
|
125
|
+
# For example, if the event has @type == "foo" and @source == "bar"
|
126
|
+
# then this string:
|
127
|
+
# "type is ${@type} and source is #{@source}"
|
128
|
+
# will return
|
129
|
+
# "type is foo and source is bar"
|
130
|
+
#
|
131
|
+
# If a ${name} value does not exist, then no substitution occurs.
|
132
|
+
#
|
133
|
+
# TODO(sissel): It is not clear what the value of a field that
|
134
|
+
# is an array (or hash?) should be. Join by comma? Something else?
|
135
|
+
public
|
136
|
+
def sprintf(format)
|
137
|
+
result = format.gsub(/\$\{[@A-Za-z0-9_-]+\}/) do |match|
|
138
|
+
name = match[2..-2] # trim '${' and '}'
|
139
|
+
value = (self[name] or match)
|
140
|
+
end
|
141
|
+
#$stderr.puts "sprintf(#{format.inspect}) => #{result.inspect}"
|
142
|
+
return result
|
143
|
+
end # def sprintf
|
106
144
|
end # class LogStash::Event
|
@@ -67,7 +67,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
|
|
67
67
|
matched = false
|
68
68
|
config.each do |match|
|
69
69
|
if ! match["match"]
|
70
|
-
@
|
70
|
+
@logger.debug(["Skipping match object, no match key", match])
|
71
71
|
next
|
72
72
|
end
|
73
73
|
|
@@ -75,9 +75,12 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
|
|
75
75
|
# apply any fields/tags.
|
76
76
|
match_count = 0
|
77
77
|
match["match"].each do |field, re|
|
78
|
-
|
78
|
+
if !event[field]
|
79
|
+
@logger.debug(["Skipping match object, field not present", field, event, event[field]])
|
80
|
+
next
|
81
|
+
end
|
79
82
|
|
80
|
-
if event[field].
|
83
|
+
if event[field].nil? and match["negate"] == true
|
81
84
|
match_count += 1
|
82
85
|
end
|
83
86
|
(event[field].is_a?(Array) ? event[field] : [event[field]]).each do |value|
|
@@ -86,6 +89,7 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
|
|
86
89
|
next if re.match(value)
|
87
90
|
@logger.debug(["grep not-matched (negate requsted)", { field => value }])
|
88
91
|
else
|
92
|
+
@logger.debug(["trying regex", re, value])
|
89
93
|
next unless re.match(value)
|
90
94
|
@logger.debug(["grep matched", { field => value }])
|
91
95
|
end
|
@@ -101,14 +105,14 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
|
|
101
105
|
if match["add_fields"]
|
102
106
|
match["add_fields"].each do |field, value|
|
103
107
|
event[field] ||= []
|
104
|
-
event[field] << value
|
108
|
+
event[field] << event.sprintf(value)
|
105
109
|
@logger.debug("grep: adding #{value} to field #{field}")
|
106
110
|
end
|
107
111
|
end # if match["add_fields"]
|
108
112
|
|
109
113
|
if match["add_tags"]
|
110
114
|
match["add_tags"].each do |tag|
|
111
|
-
event.tags << tag
|
115
|
+
event.tags << event.sprintf(tag)
|
112
116
|
@logger.debug("grep: adding tag #{tag}")
|
113
117
|
end
|
114
118
|
end # if match["add_tags"]
|
@@ -17,6 +17,7 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
|
|
17
17
|
# - multiline:
|
18
18
|
# <type>:
|
19
19
|
# pattern: <regexp>
|
20
|
+
# negate: true
|
20
21
|
# what: next
|
21
22
|
# <type>
|
22
23
|
# pattern: <regexp>
|
@@ -28,6 +29,10 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
|
|
28
29
|
# The 'what' must be "previous" or "next" and indicates the relation
|
29
30
|
# to the multi-line event.
|
30
31
|
#
|
32
|
+
# The 'negate' can be "true" or "false" (defaults false). If true, a
|
33
|
+
# message not matching the pattern will constitute a match of the multiline
|
34
|
+
# filter and the what will be applied. (vice-versa is also true)
|
35
|
+
#
|
31
36
|
# For example, java stack traces are multiline and usually have the message
|
32
37
|
# starting at the far-left, then each subsequent line indented. Do this:
|
33
38
|
#
|
@@ -96,7 +101,10 @@ class LogStash::Filters::Multiline < LogStash::Filters::Base
|
|
96
101
|
key = [event.source, event.type]
|
97
102
|
pending = @pending[key]
|
98
103
|
|
99
|
-
@logger.debug(["Reg: ", typeconfig["pattern"], event.message, match])
|
104
|
+
@logger.debug(["Reg: ", typeconfig["pattern"], event.message, match, typeconfig["negate"]])
|
105
|
+
# Add negate option
|
106
|
+
match = (match and !typeconfig["negate"]) || (!match and typeconfig["negate"])
|
107
|
+
|
100
108
|
case typeconfig["what"]
|
101
109
|
when "previous"
|
102
110
|
if match
|
@@ -50,6 +50,7 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
|
|
50
50
|
end
|
51
51
|
indexmap_req.errback do
|
52
52
|
@logger.warn(["Failure configuring index", @esurl.to_s, indexmap])
|
53
|
+
raise "Failure configuring index: #{@esurl.to_s}"
|
53
54
|
end
|
54
55
|
end # def register
|
55
56
|
|
@@ -124,6 +125,7 @@ class LogStash::Outputs::Elasticsearch < LogStash::Outputs::Base
|
|
124
125
|
req.errback do
|
125
126
|
$stderr.puts "Request to index to #{@url.to_s} failed (will retry, #{tries} tries left). Event was #{event.to_s}"
|
126
127
|
EventMachine::add_timer(2) do
|
128
|
+
# TODO(sissel): Actually abort if we retry too many times.
|
127
129
|
receive_http(event, tries - 1)
|
128
130
|
end
|
129
131
|
end
|
@@ -10,8 +10,11 @@
|
|
10
10
|
if (query == undefined || query == "") {
|
11
11
|
return;
|
12
12
|
}
|
13
|
+
//console.log("Searching: " + query);
|
14
|
+
|
13
15
|
var display_query = query.replace("<", "<").replace(">", ">")
|
14
16
|
$("#querystatus").html("Loading query '" + display_query + "'")
|
17
|
+
//console.log(logstash.params)
|
15
18
|
logstash.params.q = query;
|
16
19
|
document.location.hash = escape(JSON.stringify(logstash.params));
|
17
20
|
$("#results").load("/search/ajax", logstash.params);
|
@@ -19,12 +22,22 @@
|
|
19
22
|
}, /* search */
|
20
23
|
|
21
24
|
parse_params: function(href) {
|
22
|
-
var
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
var query = href.replace(/^[^?]*\?/, "");
|
26
|
+
if (query == href) {
|
27
|
+
//console.log("No query params in link " + href);
|
28
|
+
/* No query params */
|
29
|
+
return {};
|
30
|
+
}
|
31
|
+
|
32
|
+
//console.log({ "query": query });
|
33
|
+
var param_list = query.split("&");
|
34
|
+
params = {};
|
35
|
+
//console.log({ "Parsed params": params });
|
36
|
+
for (var p in param_list) {
|
37
|
+
var a = param_list[p].split("=");
|
38
|
+
var key = a[0];
|
39
|
+
var value = a[1];
|
40
|
+
params[key] = unescape(value);
|
28
41
|
}
|
29
42
|
return params;
|
30
43
|
},
|
@@ -89,6 +102,7 @@
|
|
89
102
|
} else {
|
90
103
|
/* No hash. See if there's a query param. */
|
91
104
|
var params = logstash.parse_params(location.href);
|
105
|
+
//console.log(params)
|
92
106
|
for (var p in params) {
|
93
107
|
logstash.params[p] = params[p];
|
94
108
|
}
|
@@ -104,9 +118,10 @@
|
|
104
118
|
}
|
105
119
|
});
|
106
120
|
|
107
|
-
$("a.pager").live("click", function() {
|
121
|
+
$("a.pager, a.querychanger").live("click", function() {
|
122
|
+
/* TODO(sissel): Allow 'control click' and 'middle click' to act normally */
|
108
123
|
var href = $(this).attr("href");
|
109
|
-
var params = logstash.parse_params(
|
124
|
+
var params = logstash.parse_params(href);
|
110
125
|
for (var p in params) {
|
111
126
|
logstash.params[p] = params[p];
|
112
127
|
}
|
@@ -114,16 +129,6 @@
|
|
114
129
|
return false;
|
115
130
|
});
|
116
131
|
|
117
|
-
$("a.querychanger").live("click", function() {
|
118
|
-
var href = $(this).attr("href");
|
119
|
-
var re = new RegExp("[&?]q=([^&]+)");
|
120
|
-
var match = re.exec(href);
|
121
|
-
if (match) {
|
122
|
-
logstash.search(match[1]);
|
123
|
-
}
|
124
|
-
return false;
|
125
|
-
});
|
126
|
-
|
127
132
|
var result_row_selector = "table.results tr.event";
|
128
133
|
$(result_row_selector).live("click", function() {
|
129
134
|
var data = eval($("td.message", this).data("full"));
|
data/lib/logstash/web/server.rb
CHANGED
@@ -32,7 +32,7 @@ class LogStash::Web::Server < Sinatra::Base
|
|
32
32
|
end # '/'
|
33
33
|
|
34
34
|
aget '/search' do
|
35
|
-
result_callback = proc do
|
35
|
+
result_callback = proc do
|
36
36
|
status 500 if @error
|
37
37
|
|
38
38
|
params[:format] ||= "html"
|
@@ -104,7 +104,7 @@ class LogStash::Web::Server < Sinatra::Base
|
|
104
104
|
if count and offset
|
105
105
|
if @total > (count + offset)
|
106
106
|
@result_end = (count + offset)
|
107
|
-
else
|
107
|
+
else
|
108
108
|
@result_end = @total
|
109
109
|
end
|
110
110
|
@result_start = offset
|
@@ -137,27 +137,39 @@ class LogStash::Web::Server < Sinatra::Base
|
|
137
137
|
end # class LogStash::Web::Server
|
138
138
|
|
139
139
|
require "optparse"
|
140
|
-
Settings = Struct.new(:daemonize, :logfile)
|
140
|
+
Settings = Struct.new(:daemonize, :logfile, :address, :port)
|
141
141
|
settings = Settings.new
|
142
|
+
|
143
|
+
settings.address = "0.0.0.0"
|
144
|
+
settings.port = 9292
|
145
|
+
|
142
146
|
progname = File.basename($0)
|
143
147
|
|
144
148
|
opts = OptionParser.new do |opts|
|
145
149
|
opts.banner = "Usage: #{progname} [options]"
|
146
150
|
|
147
|
-
opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do
|
151
|
+
opts.on("-d", "--daemonize", "Daemonize (default is run in foreground).") do
|
148
152
|
settings.daemonize = true
|
149
153
|
end
|
150
154
|
|
151
155
|
opts.on("-l", "--log FILE", "Log to a given path. Default is stdout.") do |path|
|
152
156
|
settings.logfile = path
|
153
157
|
end
|
158
|
+
|
159
|
+
opts.on("-a", "--address ADDRESS", "Address on which to start webserver. Default is 0.0.0.0.") do |address|
|
160
|
+
settings.address = address
|
161
|
+
end
|
162
|
+
|
163
|
+
opts.on("-p", "--port PORT", "Port on which to start webserver. Default is 9292.") do |port|
|
164
|
+
settings.port = port.to_i
|
165
|
+
end
|
154
166
|
end
|
155
167
|
|
156
168
|
opts.parse!
|
157
169
|
|
158
170
|
if settings.daemonize
|
159
171
|
if Process.fork == nil
|
160
|
-
Process.setsid
|
172
|
+
Process.setsid
|
161
173
|
else
|
162
174
|
exit(0)
|
163
175
|
end
|
@@ -168,7 +180,7 @@ if settings.logfile
|
|
168
180
|
STDOUT.reopen(logfile)
|
169
181
|
STDERR.reopen(logfile)
|
170
182
|
elsif settings.daemonize
|
171
|
-
# Write to /dev/null if
|
183
|
+
# Write to /dev/null if
|
172
184
|
devnull = File.open("/dev/null", "w")
|
173
185
|
STDOUT.reopen(devnull)
|
174
186
|
STDERR.reopen(devnull)
|
@@ -178,4 +190,4 @@ Rack::Handler::Thin.run(
|
|
178
190
|
Rack::CommonLogger.new( \
|
179
191
|
Rack::ShowExceptions.new( \
|
180
192
|
LogStash::Web::Server.new)),
|
181
|
-
:Port =>
|
193
|
+
:Port => settings.port, :Host => settings.address)
|
data/patterns/grok-patterns
CHANGED
@@ -7,6 +7,7 @@ BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
|
|
7
7
|
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
|
8
8
|
|
9
9
|
POSINT \b(?:[0-9]+)\b
|
10
|
+
TWODIGITINT [0-9]{2}
|
10
11
|
WORD \b\w+\b
|
11
12
|
NOTSPACE \S+
|
12
13
|
DATA .*?
|
@@ -88,3 +89,10 @@ QS %{QUOTEDSTRING}
|
|
88
89
|
# Log formats
|
89
90
|
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
|
90
91
|
COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
|
92
|
+
|
93
|
+
#
|
94
|
+
# Custom formats
|
95
|
+
# Add additional custom patterns below
|
96
|
+
DATESTAMP_RAILS %{DAY} %{MONTH} %{MONTHDAY} %{TIME} (?:%{INT:ZONE} )?%{YEAR}
|
97
|
+
DATESTAMP_MYSQL %{TWODIGITINT:year}%{TWODIGITINT:month}%{TWODIGITINT:day}\s+%{TIME}
|
98
|
+
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-lite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 40220406260823
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 20110203130400
|
10
|
+
version: 0.2.20110203130400
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Jordan Sissel
|
@@ -16,7 +16,7 @@ autorequire:
|
|
16
16
|
bindir: bin
|
17
17
|
cert_chain: []
|
18
18
|
|
19
|
-
date: 2011-
|
19
|
+
date: 2011-02-03 00:00:00 -08:00
|
20
20
|
default_executable:
|
21
21
|
dependencies:
|
22
22
|
- !ruby/object:Gem::Dependency
|
@@ -221,7 +221,6 @@ files:
|
|
221
221
|
- examples/sample-agent-in-ruby.rb
|
222
222
|
- etc/tograylog.yaml
|
223
223
|
- etc/logstash-elasticsearch-rabbitmq-river.yaml
|
224
|
-
- etc/foo.yaml
|
225
224
|
- etc/init/logstash
|
226
225
|
- etc/logstash-nagios.yaml
|
227
226
|
- etc/logstash-reader.yaml
|