logstash-event 1.1.5 → 1.2.02
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +1 -1
- data/lib/logstash-event.rb +0 -2
- data/lib/logstash/event.rb +176 -187
- data/lib/logstash/namespace.rb +2 -1
- data/lib/logstash/util.rb +105 -0
- data/lib/logstash/util/fieldreference.rb +49 -0
- data/spec/event.rb +139 -0
- metadata +71 -6
- data/lib/logstash/time.rb +0 -32
- data/lib/logstash/version.rb +0 -5
data/LICENSE
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
Copyright 2009-
|
1
|
+
Copyright 2009-2013 Jordan Sissel, Pete Fritchman, and contributors.
|
2
2
|
|
3
3
|
Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
you may not use this file except in compliance with the License.
|
data/lib/logstash-event.rb
CHANGED
data/lib/logstash/event.rb
CHANGED
@@ -1,45 +1,85 @@
|
|
1
1
|
require "json"
|
2
|
-
require "logstash/time"
|
3
|
-
require "logstash/namespace"
|
4
|
-
require "uri"
|
5
2
|
require "time"
|
3
|
+
require "date"
|
4
|
+
require "logstash/namespace"
|
5
|
+
require "logstash/util/fieldreference"
|
6
|
+
|
7
|
+
# Use a custom serialization for jsonifying Time objects.
|
8
|
+
# TODO(sissel): Put this in a separate file.
|
9
|
+
class Time
|
10
|
+
def to_json(*args)
|
11
|
+
return iso8601(3).to_json(*args)
|
12
|
+
end
|
13
|
+
|
14
|
+
def inspect
|
15
|
+
return to_json
|
16
|
+
end
|
17
|
+
end
|
6
18
|
|
7
|
-
#
|
8
|
-
#
|
19
|
+
# the logstash event object.
|
20
|
+
#
|
21
|
+
# An event is simply a tuple of (timestamp, data).
|
22
|
+
# The 'timestamp' is an ISO8601 timestamp. Data is anything - any message,
|
23
|
+
# context, references, etc that are relevant to this event.
|
24
|
+
#
|
25
|
+
# Internally, this is represented as a hash with only two guaranteed fields.
|
26
|
+
#
|
27
|
+
# * "@timestamp" - an ISO8601 timestamp representing the time the event
|
28
|
+
# occurred at.
|
29
|
+
# * "@version" - the version of the schema. Currently "1"
|
30
|
+
#
|
31
|
+
# They are prefixed with an "@" symbol to avoid clashing with your
|
32
|
+
# own custom fields.
|
33
|
+
#
|
34
|
+
# When serialized, this is represented in JSON. For example:
|
9
35
|
#
|
10
|
-
#
|
11
|
-
#
|
36
|
+
# {
|
37
|
+
# "@timestamp": "2013-02-09T20:39:26.234Z",
|
38
|
+
# "@version": "1",
|
39
|
+
# message: "hello world"
|
40
|
+
# }
|
12
41
|
class LogStash::Event
|
42
|
+
class DeprecatedMethod < StandardError; end
|
43
|
+
|
13
44
|
public
|
14
|
-
def initialize(data=
|
45
|
+
def initialize(data={})
|
15
46
|
@cancelled = false
|
16
47
|
|
17
|
-
@data =
|
18
|
-
|
19
|
-
"@
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
48
|
+
@data = data
|
49
|
+
if data.include?("@timestamp")
|
50
|
+
t = data["@timestamp"]
|
51
|
+
if t.is_a?(String)
|
52
|
+
data["@timestamp"] = Time.parse(t).gmtime
|
53
|
+
end
|
54
|
+
else
|
55
|
+
data["@timestamp"] = ::Time.now.utc
|
56
|
+
end
|
57
|
+
data["@version"] = "1" if !@data.include?("@version")
|
24
58
|
end # def initialize
|
25
59
|
|
26
|
-
|
27
|
-
@@date_parser = Java::org.joda.time.format.ISODateTimeFormat.dateTimeParser.withOffsetParsed
|
28
|
-
else
|
29
|
-
# TODO(sissel): LOGSTASH-217
|
30
|
-
@@date_parser ||= nil
|
31
|
-
end
|
32
|
-
|
60
|
+
# Add class methods on inclusion.
|
33
61
|
public
|
34
|
-
def self.
|
35
|
-
|
36
|
-
end # def
|
62
|
+
def self.included(klass)
|
63
|
+
klass.extend(ClassMethods)
|
64
|
+
end # def included
|
65
|
+
|
66
|
+
module ClassMethods
|
67
|
+
public
|
68
|
+
def from_json(json)
|
69
|
+
return self.new(JSON.parse(json))
|
70
|
+
end # def from_json
|
71
|
+
end
|
37
72
|
|
38
73
|
public
|
39
74
|
def cancel
|
40
75
|
@cancelled = true
|
41
76
|
end # def cancel
|
42
77
|
|
78
|
+
public
|
79
|
+
def uncancel
|
80
|
+
@cancelled = false
|
81
|
+
end # def uncancel
|
82
|
+
|
43
83
|
public
|
44
84
|
def cancelled?
|
45
85
|
return @cancelled
|
@@ -48,95 +88,90 @@ class LogStash::Event
|
|
48
88
|
# Create a deep-ish copy of this event.
|
49
89
|
public
|
50
90
|
def clone
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
91
|
+
copy = {}
|
92
|
+
@data.each do |k,v|
|
93
|
+
# TODO(sissel): Recurse if this is a hash/array?
|
94
|
+
copy[k] = v.clone
|
55
95
|
end
|
56
|
-
return
|
96
|
+
return self.class.new(copy)
|
57
97
|
end # def clone
|
58
98
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
def unix_timestamp
|
70
|
-
if RUBY_ENGINE != "jruby"
|
71
|
-
# This is really slow. See LOGSTASH-217
|
72
|
-
return Time.parse(timestamp).to_f
|
73
|
-
else
|
74
|
-
time = @@date_parser.parseDateTime(timestamp)
|
75
|
-
return time.getMillis.to_f / 1000
|
76
|
-
end
|
99
|
+
if RUBY_ENGINE == "jruby"
|
100
|
+
public
|
101
|
+
def to_s
|
102
|
+
return self.sprintf("%{+yyyy-MM-dd'T'HH:mm:ss.SSSZ} %{host} %{message}")
|
103
|
+
end # def to_s
|
104
|
+
else
|
105
|
+
public
|
106
|
+
def to_s
|
107
|
+
return self.sprintf("#{self["@timestamp"].iso8601} %{host} %{message}")
|
108
|
+
end # def to_s
|
77
109
|
end
|
78
110
|
|
79
111
|
public
|
80
|
-
def
|
81
|
-
def
|
82
|
-
uri = URI.parse(val) rescue nil
|
83
|
-
val = uri if uri
|
84
|
-
if val.is_a?(URI)
|
85
|
-
@data["@source"] = val.to_s
|
86
|
-
@data["@source_host"] = val.host
|
87
|
-
@data["@source_path"] = val.path
|
88
|
-
else
|
89
|
-
@data["@source"] = val
|
90
|
-
@data["@source_host"] = val
|
91
|
-
end
|
92
|
-
end # def source=
|
93
|
-
|
94
|
-
public
|
95
|
-
def source_host; @data["@source_host"]; end # def source_host
|
96
|
-
def source_host=(val); @data["@source_host"] = val; end # def source_host=
|
97
|
-
|
98
|
-
public
|
99
|
-
def source_path; @data["@source_path"]; end # def source_path
|
100
|
-
def source_path=(val); @data["@source_path"] = val; end # def source_path=
|
101
|
-
|
102
|
-
public
|
103
|
-
def message; @data["@message"]; end # def message
|
104
|
-
def message=(val); @data["@message"] = val; end # def message=
|
105
|
-
|
106
|
-
public
|
107
|
-
def type; @data["@type"]; end # def type
|
108
|
-
def type=(val); @data["@type"] = val; end # def type=
|
112
|
+
def timestamp; return @data["@timestamp"]; end # def timestamp
|
113
|
+
def timestamp=(val); return @data["@timestamp"] = val; end # def timestamp=
|
109
114
|
|
110
|
-
|
111
|
-
|
112
|
-
|
115
|
+
def unix_timestamp
|
116
|
+
raise DeprecatedMethod
|
117
|
+
end # def unix_timestamp
|
113
118
|
|
119
|
+
def ruby_timestamp
|
120
|
+
raise DeprecatedMethod
|
121
|
+
end # def unix_timestamp
|
122
|
+
|
114
123
|
# field-related access
|
115
124
|
public
|
116
|
-
def [](
|
117
|
-
|
118
|
-
if ! @data["@fields"].has_key?(key) and key.slice(0,1) == "@"
|
119
|
-
return @data[key]
|
120
|
-
# Exists in @fields (returns value) or doesn't start with "@" (return null)
|
125
|
+
def [](str)
|
126
|
+
if str[0,1] == "+"
|
121
127
|
else
|
122
|
-
return @data
|
128
|
+
return LogStash::Util::FieldReference.exec(str, @data)
|
123
129
|
end
|
124
130
|
end # def []
|
125
131
|
|
126
132
|
public
|
127
|
-
def []=(
|
128
|
-
|
129
|
-
|
130
|
-
else
|
131
|
-
@data["@fields"][key] = value
|
133
|
+
def []=(str, value)
|
134
|
+
r = LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
|
135
|
+
obj[key] = value
|
132
136
|
end
|
137
|
+
|
138
|
+
# The assignment can fail if the given field reference (str) does not exist
|
139
|
+
# In this case, we'll want to set the value manually.
|
140
|
+
if r.nil?
|
141
|
+
# TODO(sissel): Implement this in LogStash::Util::FieldReference
|
142
|
+
if str[0,1] != "["
|
143
|
+
return @data[str] = value
|
144
|
+
end
|
145
|
+
|
146
|
+
# No existing element was found, so let's set one.
|
147
|
+
*parents, key = str.scan(/(?<=\[)[^\]]+(?=\])/)
|
148
|
+
obj = @data
|
149
|
+
parents.each do |p|
|
150
|
+
if obj.include?(p)
|
151
|
+
obj = obj[p]
|
152
|
+
else
|
153
|
+
obj[p] = {}
|
154
|
+
obj = obj[p]
|
155
|
+
end
|
156
|
+
end
|
157
|
+
obj[key] = value
|
158
|
+
end
|
159
|
+
return value
|
133
160
|
end # def []=
|
134
161
|
|
135
|
-
|
162
|
+
public
|
163
|
+
def fields
|
164
|
+
raise DeprecatedMethod
|
165
|
+
end
|
136
166
|
|
137
167
|
public
|
138
|
-
def to_json(*args)
|
139
|
-
|
168
|
+
def to_json(*args)
|
169
|
+
return @data.to_json(*args)
|
170
|
+
end # def to_json
|
171
|
+
|
172
|
+
def to_hash
|
173
|
+
return @data
|
174
|
+
end # def to_hash
|
140
175
|
|
141
176
|
public
|
142
177
|
def overwrite(event)
|
@@ -145,45 +180,22 @@ class LogStash::Event
|
|
145
180
|
|
146
181
|
public
|
147
182
|
def include?(key)
|
148
|
-
return
|
183
|
+
return !self[key].nil?
|
149
184
|
end # def include?
|
150
185
|
|
151
186
|
# Append an event to this one.
|
152
187
|
public
|
153
188
|
def append(event)
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
else
|
158
|
-
self.message = event.message
|
159
|
-
end
|
160
|
-
end
|
161
|
-
self.tags |= event.tags
|
189
|
+
# non-destructively merge that event with ourselves.
|
190
|
+
LogStash::Util.hash_merge(@data, event.to_hash)
|
191
|
+
end # append
|
162
192
|
|
163
|
-
|
164
|
-
|
165
|
-
if self.fields.include?(name)
|
166
|
-
if !self.fields[name].is_a?(Array)
|
167
|
-
self.fields[name] = [self.fields[name]]
|
168
|
-
end
|
169
|
-
if value.is_a?(Array)
|
170
|
-
self.fields[name] |= value
|
171
|
-
else
|
172
|
-
self.fields[name] << value unless self.fields[name].include?(value)
|
173
|
-
end
|
174
|
-
else
|
175
|
-
self.fields[name] = value
|
176
|
-
end
|
177
|
-
end # event.fields.each
|
178
|
-
end # def append
|
179
|
-
|
180
|
-
# Remove a field
|
193
|
+
# Remove a field or field reference. Returns the value of that field when
|
194
|
+
# deleted
|
181
195
|
public
|
182
|
-
def remove(
|
183
|
-
|
184
|
-
|
185
|
-
else
|
186
|
-
@data["@fields"].delete(field)
|
196
|
+
def remove(str)
|
197
|
+
return LogStash::Util::FieldReference.exec(str, @data) do |obj, key|
|
198
|
+
next obj.delete(key)
|
187
199
|
end
|
188
200
|
end # def remove
|
189
201
|
|
@@ -192,9 +204,9 @@ class LogStash::Event
|
|
192
204
|
# any format values, delimited by %{foo} where 'foo' is a field or
|
193
205
|
# metadata member.
|
194
206
|
#
|
195
|
-
# For example, if the event has
|
207
|
+
# For example, if the event has type == "foo" and source == "bar"
|
196
208
|
# then this string:
|
197
|
-
# "type is %{
|
209
|
+
# "type is %{type} and source is %{host}"
|
198
210
|
# will return
|
199
211
|
# "type is foo and source is bar"
|
200
212
|
#
|
@@ -205,6 +217,7 @@ class LogStash::Event
|
|
205
217
|
# is an array (or hash?) should be. Join by comma? Something else?
|
206
218
|
public
|
207
219
|
def sprintf(format)
|
220
|
+
format = format.to_s
|
208
221
|
if format.index("%").nil?
|
209
222
|
return format
|
210
223
|
end
|
@@ -215,69 +228,45 @@ class LogStash::Event
|
|
215
228
|
|
216
229
|
if key == "+%s"
|
217
230
|
# Got %{+%s}, support for unix epoch time
|
218
|
-
|
219
|
-
# This is really slow. See LOGSTASH-217
|
220
|
-
Date.parse(self.timestamp).to_i
|
221
|
-
else
|
222
|
-
datetime = @@date_parser.parseDateTime(self.timestamp)
|
223
|
-
(datetime.getMillis / 1000).to_i
|
224
|
-
end
|
231
|
+
next @data["@timestamp"].to_i
|
225
232
|
elsif key[0,1] == "+"
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
datetime.toString(format) # return requested time format
|
236
|
-
end
|
233
|
+
t = @data["@timestamp"]
|
234
|
+
formatter = org.joda.time.format.DateTimeFormat.forPattern(key[1 .. -1])\
|
235
|
+
.withZone(org.joda.time.DateTimeZone::UTC)
|
236
|
+
#next org.joda.time.Instant.new(t.tv_sec * 1000 + t.tv_usec / 1000).toDateTime.toString(formatter)
|
237
|
+
# Invoke a specific Instant constructor to avoid this warning in JRuby
|
238
|
+
# > ambiguous Java methods found, using org.joda.time.Instant(long)
|
239
|
+
org.joda.time.Instant.java_class.constructor(Java::long).new_instance(
|
240
|
+
t.tv_sec * 1000 + t.tv_usec / 1000
|
241
|
+
).to_java.toDateTime.toString(formatter)
|
237
242
|
else
|
238
|
-
|
239
|
-
value = nil
|
240
|
-
obj = self
|
241
|
-
|
242
|
-
# If the top-level value exists, use that and don't try
|
243
|
-
# to "look" into data structures.
|
244
|
-
if self[key]
|
245
|
-
value = self[key]
|
246
|
-
else
|
247
|
-
# "." is what ES uses to access structured data, so adopt that
|
248
|
-
# idea here, too. "foo.bar" will access key "bar" under hash "foo".
|
249
|
-
key.split('.').each do |segment|
|
250
|
-
if obj
|
251
|
-
value = obj[segment] rescue nil
|
252
|
-
obj = obj[segment] rescue nil
|
253
|
-
else
|
254
|
-
value = nil
|
255
|
-
break
|
256
|
-
end
|
257
|
-
end # key.split.each
|
258
|
-
end # if self[key]
|
259
|
-
|
243
|
+
value = self[key]
|
260
244
|
case value
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
end
|
270
|
-
end
|
271
|
-
end
|
245
|
+
when nil
|
246
|
+
tok # leave the %{foo} if this field does not exist in this event.
|
247
|
+
when Array
|
248
|
+
value.join(",") # Join by ',' if value is an array
|
249
|
+
when Hash
|
250
|
+
value.to_json # Convert hashes to json
|
251
|
+
else
|
252
|
+
value # otherwise return the value
|
253
|
+
end # case value
|
254
|
+
end # 'key' checking
|
255
|
+
end # format.gsub...
|
272
256
|
end # def sprintf
|
273
257
|
|
274
|
-
|
275
|
-
def
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
258
|
+
# Shims to remove after event v1 is the default.
|
259
|
+
def tags=(value); self["tags"] = value; end
|
260
|
+
def tags; return self["tags"]; end
|
261
|
+
def message=(value); self["message"] = value; end
|
262
|
+
def source=(value); self["source"] = value; end
|
263
|
+
def type=(value); self["type"] = value; end
|
264
|
+
def type; return self["type"]; end
|
265
|
+
def fields; return self.to_hash; end
|
280
266
|
|
281
|
-
|
282
|
-
|
267
|
+
def tag(value)
|
268
|
+
# Generalize this method for more usability
|
269
|
+
self["tags"] ||= []
|
270
|
+
self["tags"] << value unless self["tags"].include?(value)
|
271
|
+
end
|
283
272
|
end # class LogStash::Event
|
data/lib/logstash/namespace.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
#$: << File.join(File.dirname(__FILE__), "..", "..", "vendor", "bundle")
|
2
2
|
|
3
3
|
module LogStash
|
4
4
|
module Inputs; end
|
@@ -9,6 +9,7 @@ module LogStash
|
|
9
9
|
module File; end
|
10
10
|
module Web; end
|
11
11
|
module Util; end
|
12
|
+
module PluginMixins; end
|
12
13
|
|
13
14
|
SHUTDOWN = :shutdown
|
14
15
|
end # module LogStash
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require "logstash/namespace"
|
2
|
+
|
3
|
+
module LogStash::Util
|
4
|
+
UNAME = case RbConfig::CONFIG["host_os"]
|
5
|
+
when /^linux/; "linux"
|
6
|
+
else; RbConfig::CONFIG["host_os"]
|
7
|
+
end
|
8
|
+
|
9
|
+
PR_SET_NAME = 15
|
10
|
+
def self.set_thread_name(name)
|
11
|
+
if RUBY_ENGINE == "jruby"
|
12
|
+
# Keep java and ruby thread names in sync.
|
13
|
+
Java::java.lang.Thread.currentThread.setName(name)
|
14
|
+
end
|
15
|
+
Thread.current[:name] = name
|
16
|
+
|
17
|
+
if UNAME == "linux"
|
18
|
+
require "logstash/util/prctl"
|
19
|
+
# prctl PR_SET_NAME allows up to 16 bytes for a process name
|
20
|
+
# since MRI 1.9, JRuby, and Rubinius use system threads for this.
|
21
|
+
LibC.prctl(PR_SET_NAME, name[0..16], 0, 0, 0)
|
22
|
+
end
|
23
|
+
end # def set_thread_name
|
24
|
+
|
25
|
+
# Merge hash 'src' into 'dst' nondestructively
|
26
|
+
#
|
27
|
+
# Duplicate keys will become array values
|
28
|
+
#
|
29
|
+
# [ src["foo"], dst["foo"] ]
|
30
|
+
def self.hash_merge(dst, src)
|
31
|
+
src.each do |name, svalue|
|
32
|
+
if dst.include?(name)
|
33
|
+
dvalue = dst[name]
|
34
|
+
if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
|
35
|
+
dvalue = hash_merge(dvalue, svalue)
|
36
|
+
elsif svalue.is_a?(Array)
|
37
|
+
if dvalue.is_a?(Array)
|
38
|
+
# merge arrays without duplicates.
|
39
|
+
dvalue |= svalue
|
40
|
+
else
|
41
|
+
dvalue = [dvalue] | svalue
|
42
|
+
end
|
43
|
+
else
|
44
|
+
if dvalue.is_a?(Array)
|
45
|
+
dvalue << svalue unless dvalue.include?(svalue)
|
46
|
+
else
|
47
|
+
dvalue = [dvalue, svalue] unless dvalue == svalue
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
dst[name] = dvalue
|
52
|
+
else
|
53
|
+
# dst doesn't have this key, just set it.
|
54
|
+
dst[name] = svalue
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
return dst
|
59
|
+
end # def self.hash_merge
|
60
|
+
|
61
|
+
# Merge hash 'src' into 'dst' nondestructively
|
62
|
+
#
|
63
|
+
# Duplicate keys will become array values
|
64
|
+
# Arrays merged will simply be appended.
|
65
|
+
#
|
66
|
+
# [ src["foo"], dst["foo"] ]
|
67
|
+
def self.hash_merge_with_dups(dst, src)
|
68
|
+
src.each do |name, svalue|
|
69
|
+
if dst.include?(name)
|
70
|
+
dvalue = dst[name]
|
71
|
+
if dvalue.is_a?(Hash) && svalue.is_a?(Hash)
|
72
|
+
dvalue = hash_merge(dvalue, svalue)
|
73
|
+
elsif svalue.is_a?(Array)
|
74
|
+
if dvalue.is_a?(Array)
|
75
|
+
# merge arrays without duplicates.
|
76
|
+
dvalue += svalue
|
77
|
+
else
|
78
|
+
dvalue = [dvalue] + svalue
|
79
|
+
end
|
80
|
+
else
|
81
|
+
if dvalue.is_a?(Array)
|
82
|
+
dvalue << svalue unless dvalue.include?(svalue)
|
83
|
+
else
|
84
|
+
dvalue = [dvalue, svalue] unless dvalue == svalue
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
dst[name] = dvalue
|
89
|
+
else
|
90
|
+
# dst doesn't have this key, just set it.
|
91
|
+
dst[name] = svalue
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
return dst
|
96
|
+
end # def self.hash_merge
|
97
|
+
|
98
|
+
def self.hash_merge_many(*hashes)
|
99
|
+
dst = {}
|
100
|
+
hashes.each do |hash|
|
101
|
+
hash_merge_with_dups(dst, hash)
|
102
|
+
end
|
103
|
+
return dst
|
104
|
+
end # def hash_merge_many
|
105
|
+
end # module LogStash::Util
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require "logstash/namespace"
|
2
|
+
require "logstash/util"
|
3
|
+
|
4
|
+
module LogStash::Util::FieldReference
|
5
|
+
def compile(str)
|
6
|
+
if str[0,1] != '['
|
7
|
+
return <<-"CODE"
|
8
|
+
lambda do |e, &block|
|
9
|
+
return block.call(e, #{str.inspect}) unless block.nil?
|
10
|
+
return e[#{str.inspect}]
|
11
|
+
end
|
12
|
+
CODE
|
13
|
+
end
|
14
|
+
|
15
|
+
code = "lambda do |e, &block|\n"
|
16
|
+
selectors = str.scan(/(?<=\[).+?(?=\])/)
|
17
|
+
selectors.each_with_index do |tok, i|
|
18
|
+
last = (i == selectors.count() - 1)
|
19
|
+
code << " # [#{tok}]#{ last ? " (last selector)" : "" }\n"
|
20
|
+
|
21
|
+
if last
|
22
|
+
code << <<-"CODE"
|
23
|
+
return block.call(e, #{tok.inspect}) unless block.nil?
|
24
|
+
CODE
|
25
|
+
end
|
26
|
+
|
27
|
+
code << <<-"CODE"
|
28
|
+
if e.is_a?(Array)
|
29
|
+
e = e[#{tok.to_i}]
|
30
|
+
else
|
31
|
+
e = e[#{tok.inspect}]
|
32
|
+
end
|
33
|
+
return e if e.nil?
|
34
|
+
CODE
|
35
|
+
|
36
|
+
end
|
37
|
+
code << "return e\nend"
|
38
|
+
#puts code
|
39
|
+
return code
|
40
|
+
end # def compile
|
41
|
+
|
42
|
+
def exec(str, obj, &block)
|
43
|
+
@__fieldeval_cache ||= {}
|
44
|
+
@__fieldeval_cache[str] ||= eval(compile(str))
|
45
|
+
return @__fieldeval_cache[str].call(obj, &block)
|
46
|
+
end
|
47
|
+
|
48
|
+
extend self
|
49
|
+
end # module LogStash::Util::FieldReference
|
data/spec/event.rb
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
require "logstash/event"
|
2
|
+
require "insist"
|
3
|
+
|
4
|
+
describe LogStash::Event do
|
5
|
+
subject do
|
6
|
+
LogStash::Event.new(
|
7
|
+
"@timestamp" => Time.iso8601("2013-01-01T00:00:00.000Z"),
|
8
|
+
"type" => "sprintf",
|
9
|
+
"message" => "hello world",
|
10
|
+
"tags" => [ "tag1" ],
|
11
|
+
"source" => "/home/foo",
|
12
|
+
"a" => "b",
|
13
|
+
"c" => {
|
14
|
+
"d" => "f",
|
15
|
+
"e" => {"f" => "g"}
|
16
|
+
},
|
17
|
+
"f" => { "g" => { "h" => "i" } },
|
18
|
+
"j" => {
|
19
|
+
"k1" => "v",
|
20
|
+
"k2" => [ "w", "x" ],
|
21
|
+
"k3" => {"4" => "m"},
|
22
|
+
5 => 6,
|
23
|
+
"5" => 7
|
24
|
+
}
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
context "#sprintf" do
|
29
|
+
it "should report a unix timestamp for %{+%s}" do
|
30
|
+
insist { subject.sprintf("%{+%s}") } == "1356998400"
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should report a time with %{+format} syntax", :if => RUBY_ENGINE == "jruby" do
|
34
|
+
insist { subject.sprintf("%{+YYYY}") } == "2013"
|
35
|
+
insist { subject.sprintf("%{+MM}") } == "01"
|
36
|
+
insist { subject.sprintf("%{+HH}") } == "00"
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should report fields with %{field} syntax" do
|
40
|
+
insist { subject.sprintf("%{type}") } == "sprintf"
|
41
|
+
insist { subject.sprintf("%{message}") } == subject["message"]
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should print deep fields" do
|
45
|
+
insist { subject.sprintf("%{[j][k1]}") } == "v"
|
46
|
+
insist { subject.sprintf("%{[j][k2][0]}") } == "w"
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should be able to take a non-string for the format" do
|
50
|
+
insist { subject.sprintf(2) } == "2"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context "#[]" do
|
55
|
+
it "should fetch data" do
|
56
|
+
insist { subject["type"] } == "sprintf"
|
57
|
+
end
|
58
|
+
it "should fetch fields" do
|
59
|
+
insist { subject["a"] } == "b"
|
60
|
+
insist { subject['c']['d'] } == "f"
|
61
|
+
end
|
62
|
+
it "should fetch deep fields" do
|
63
|
+
insist { subject["[j][k1]"] } == "v"
|
64
|
+
insist { subject["[c][d]"] } == "f"
|
65
|
+
insist { subject['[f][g][h]'] } == "i"
|
66
|
+
insist { subject['[j][k3][4]'] } == "m"
|
67
|
+
insist { subject['[j][5]'] } == 7
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should be fast?", :if => ENV["SPEEDTEST"] do
|
72
|
+
2.times do
|
73
|
+
start = Time.now
|
74
|
+
100000.times { subject["[j][k1]"] }
|
75
|
+
puts "Duration: #{Time.now - start}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
context "#append" do
|
81
|
+
it "should append strings to an array" do
|
82
|
+
subject.append(LogStash::Event.new("message" => "another thing"))
|
83
|
+
insist { subject["message"] } == [ "hello world", "another thing" ]
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should concatenate tags" do
|
87
|
+
subject.append(LogStash::Event.new("tags" => [ "tag2" ]))
|
88
|
+
insist { subject["tags"] } == [ "tag1", "tag2" ]
|
89
|
+
end
|
90
|
+
|
91
|
+
context "when event field is nil" do
|
92
|
+
it "should add single value as string" do
|
93
|
+
subject.append(LogStash::Event.new({"field1" => "append1"}))
|
94
|
+
insist { subject[ "field1" ] } == "append1"
|
95
|
+
end
|
96
|
+
it "should add multi values as array" do
|
97
|
+
subject.append(LogStash::Event.new({"field1" => [ "append1","append2" ]}))
|
98
|
+
insist { subject[ "field1" ] } == [ "append1","append2" ]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context "when event field is a string" do
|
103
|
+
before { subject[ "field1" ] = "original1" }
|
104
|
+
|
105
|
+
it "should append string to values, if different from current" do
|
106
|
+
subject.append(LogStash::Event.new({"field1" => "append1"}))
|
107
|
+
insist { subject[ "field1" ] } == [ "original1", "append1" ]
|
108
|
+
end
|
109
|
+
it "should not change value, if appended value is equal current" do
|
110
|
+
subject.append(LogStash::Event.new({"field1" => "original1"}))
|
111
|
+
insist { subject[ "field1" ] } == "original1"
|
112
|
+
end
|
113
|
+
it "should concatenate values in an array" do
|
114
|
+
subject.append(LogStash::Event.new({"field1" => [ "append1" ]}))
|
115
|
+
insist { subject[ "field1" ] } == [ "original1", "append1" ]
|
116
|
+
end
|
117
|
+
it "should join array, removing duplicates" do
|
118
|
+
subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
|
119
|
+
insist { subject[ "field1" ] } == [ "original1", "append1" ]
|
120
|
+
end
|
121
|
+
end
|
122
|
+
context "when event field is an array" do
|
123
|
+
before { subject[ "field1" ] = [ "original1", "original2" ] }
|
124
|
+
|
125
|
+
it "should append string values to array, if not present in array" do
|
126
|
+
subject.append(LogStash::Event.new({"field1" => "append1"}))
|
127
|
+
insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
|
128
|
+
end
|
129
|
+
it "should not append string values, if the array already contains it" do
|
130
|
+
subject.append(LogStash::Event.new({"field1" => "original1"}))
|
131
|
+
insist { subject[ "field1" ] } == [ "original1", "original2" ]
|
132
|
+
end
|
133
|
+
it "should join array, removing duplicates" do
|
134
|
+
subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
|
135
|
+
insist { subject[ "field1" ] } == [ "original1", "original2", "append1" ]
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-event
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.02
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,8 +9,72 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
13
|
-
dependencies:
|
12
|
+
date: 2013-09-11 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: rspec
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: guard
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: guard-rspec
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: insist
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - '='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.0.0
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - '='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.0.0
|
14
78
|
description: Library that contains the classes required to create LogStash events
|
15
79
|
email:
|
16
80
|
- jls@semicomplete.com
|
@@ -21,8 +85,9 @@ files:
|
|
21
85
|
- lib/logstash-event.rb
|
22
86
|
- lib/logstash/event.rb
|
23
87
|
- lib/logstash/namespace.rb
|
24
|
-
- lib/logstash/
|
25
|
-
- lib/logstash/
|
88
|
+
- lib/logstash/util/fieldreference.rb
|
89
|
+
- lib/logstash/util.rb
|
90
|
+
- spec/event.rb
|
26
91
|
- LICENSE
|
27
92
|
homepage: https://github.com/logstash/logstash
|
28
93
|
licenses:
|
@@ -45,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
45
110
|
version: '0'
|
46
111
|
requirements: []
|
47
112
|
rubyforge_project:
|
48
|
-
rubygems_version: 1.8.
|
113
|
+
rubygems_version: 1.8.25
|
49
114
|
signing_key:
|
50
115
|
specification_version: 3
|
51
116
|
summary: Library that contains the classes required to create LogStash events
|
data/lib/logstash/time.rb
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
require "logstash/namespace"
|
2
|
-
|
3
|
-
# Provide our own Time wrapper for ISO8601 support
|
4
|
-
# Example:
|
5
|
-
# >> LogStash::Time.now.to_iso8601
|
6
|
-
# => "2010-10-17 00:25:24.619014-0700"
|
7
|
-
#
|
8
|
-
# >> LogStash::Time.now.utc.to_iso8601
|
9
|
-
# => "2010-10-17 07:25:26.788704Z"
|
10
|
-
module LogStash::Time
|
11
|
-
if RUBY_ENGINE == "jruby"
|
12
|
-
require "java"
|
13
|
-
DateTime = org.joda.time.DateTime
|
14
|
-
DateTimeZone = org.joda.time.DateTimeZone
|
15
|
-
def self.now
|
16
|
-
# org.joda.time.DateTime#to_s returns the time in ISO8601 form :)
|
17
|
-
# Could call DateTime.new(DateTimeZone::UTC) but JRuby calls the
|
18
|
-
# DateTime#new(Object) constructor instead of the
|
19
|
-
# DateTime#new(DateTimeZone) constructor. I was unable to get java_send to invoke this constructor,
|
20
|
-
# so instead I have to do DateTime#new#withZone(UTC)
|
21
|
-
return DateTime.new.withZone(DateTimeZone::UTC).to_s
|
22
|
-
end # def initialize
|
23
|
-
else
|
24
|
-
# Otherwise, use ruby stdlib Time, which is much slower than Joda.
|
25
|
-
ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
|
26
|
-
def self.now
|
27
|
-
now = Time.new.utc
|
28
|
-
return sprintf(ISO8601_STRFTIME, now.year, now.month, now.day, now.hour,
|
29
|
-
now.min, now.sec, now.tv_usec, now.utc_offset / 3600)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end # module LogStash::Time
|