logstash-codec-cef 6.1.2-java → 6.2.3-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -1
- data/docs/index.asciidoc +452 -9
- data/lib/logstash/codecs/cef.rb +272 -111
- data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
- data/logstash-codec-cef.gemspec +3 -1
- data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
- data/spec/codecs/cef_spec.rb +500 -306
- metadata +37 -7
data/lib/logstash/codecs/cef.rb
CHANGED
@@ -3,6 +3,10 @@ require "logstash/util/buftok"
|
|
3
3
|
require "logstash/util/charset"
|
4
4
|
require "logstash/codecs/base"
|
5
5
|
require "json"
|
6
|
+
require "time"
|
7
|
+
|
8
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
9
|
+
require 'logstash/plugin_mixins/event_support/event_factory_adapter'
|
6
10
|
|
7
11
|
# Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
8
12
|
# Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
@@ -13,6 +17,11 @@ require "json"
|
|
13
17
|
class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
14
18
|
config_name "cef"
|
15
19
|
|
20
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
21
|
+
include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
|
22
|
+
|
23
|
+
InvalidTimestamp = Class.new(StandardError)
|
24
|
+
|
16
25
|
# Device vendor field in CEF header. The new value can include `%{foo}` strings
|
17
26
|
# to help you build a new value from other parts of the event.
|
18
27
|
config :vendor, :validate => :string, :default => "Elasticsearch"
|
@@ -68,106 +77,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
68
77
|
# * `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
69
78
|
config :delimiter, :validate => :string
|
70
79
|
|
80
|
+
# When parsing timestamps that do not include a UTC offset in payloads that do not
|
81
|
+
# include the device's timezone, the default timezone is used.
|
82
|
+
# If none is provided the system timezone is used.
|
83
|
+
config :default_timezone, :validate => :string
|
84
|
+
|
85
|
+
# The locale is used to parse abbreviated month names from some CEF timestamp
|
86
|
+
# formats.
|
87
|
+
# If none is provided, the system default is used.
|
88
|
+
config :locale, :validate => :string
|
89
|
+
|
71
90
|
# If raw_data_field is set, during decode of an event an additional field with
|
72
91
|
# the provided name is added, which contains the raw data.
|
73
92
|
config :raw_data_field, :validate => :string
|
74
93
|
|
75
|
-
|
76
|
-
|
77
|
-
#
|
78
|
-
|
79
|
-
"act" => "deviceAction",
|
80
|
-
"app" => "applicationProtocol",
|
81
|
-
"c6a1" => "deviceCustomIPv6Address1",
|
82
|
-
"c6a1Label" => "deviceCustomIPv6Address1Label",
|
83
|
-
"c6a2" => "deviceCustomIPv6Address2",
|
84
|
-
"c6a2Label" => "deviceCustomIPv6Address2Label",
|
85
|
-
"c6a3" => "deviceCustomIPv6Address3",
|
86
|
-
"c6a3Label" => "deviceCustomIPv6Address3Label",
|
87
|
-
"c6a4" => "deviceCustomIPv6Address4",
|
88
|
-
"c6a4Label" => "deviceCustomIPv6Address4Label",
|
89
|
-
"cat" => "deviceEventCategory",
|
90
|
-
"cfp1" => "deviceCustomFloatingPoint1",
|
91
|
-
"cfp1Label" => "deviceCustomFloatingPoint1Label",
|
92
|
-
"cfp2" => "deviceCustomFloatingPoint2",
|
93
|
-
"cfp2Label" => "deviceCustomFloatingPoint2Label",
|
94
|
-
"cfp3" => "deviceCustomFloatingPoint3",
|
95
|
-
"cfp3Label" => "deviceCustomFloatingPoint3Label",
|
96
|
-
"cfp4" => "deviceCustomFloatingPoint4",
|
97
|
-
"cfp4Label" => "deviceCustomFloatingPoint4Label",
|
98
|
-
"cn1" => "deviceCustomNumber1",
|
99
|
-
"cn1Label" => "deviceCustomNumber1Label",
|
100
|
-
"cn2" => "deviceCustomNumber2",
|
101
|
-
"cn2Label" => "deviceCustomNumber2Label",
|
102
|
-
"cn3" => "deviceCustomNumber3",
|
103
|
-
"cn3Label" => "deviceCustomNumber3Label",
|
104
|
-
"cnt" => "baseEventCount",
|
105
|
-
"cs1" => "deviceCustomString1",
|
106
|
-
"cs1Label" => "deviceCustomString1Label",
|
107
|
-
"cs2" => "deviceCustomString2",
|
108
|
-
"cs2Label" => "deviceCustomString2Label",
|
109
|
-
"cs3" => "deviceCustomString3",
|
110
|
-
"cs3Label" => "deviceCustomString3Label",
|
111
|
-
"cs4" => "deviceCustomString4",
|
112
|
-
"cs4Label" => "deviceCustomString4Label",
|
113
|
-
"cs5" => "deviceCustomString5",
|
114
|
-
"cs5Label" => "deviceCustomString5Label",
|
115
|
-
"cs6" => "deviceCustomString6",
|
116
|
-
"cs6Label" => "deviceCustomString6Label",
|
117
|
-
"dhost" => "destinationHostName",
|
118
|
-
"dmac" => "destinationMacAddress",
|
119
|
-
"dntdom" => "destinationNtDomain",
|
120
|
-
"dpid" => "destinationProcessId",
|
121
|
-
"dpriv" => "destinationUserPrivileges",
|
122
|
-
"dproc" => "destinationProcessName",
|
123
|
-
"dpt" => "destinationPort",
|
124
|
-
"dst" => "destinationAddress",
|
125
|
-
"duid" => "destinationUserId",
|
126
|
-
"duser" => "destinationUserName",
|
127
|
-
"dvc" => "deviceAddress",
|
128
|
-
"dvchost" => "deviceHostName",
|
129
|
-
"dvcpid" => "deviceProcessId",
|
130
|
-
"end" => "endTime",
|
131
|
-
"fname" => "fileName",
|
132
|
-
"fsize" => "fileSize",
|
133
|
-
"in" => "bytesIn",
|
134
|
-
"msg" => "message",
|
135
|
-
"out" => "bytesOut",
|
136
|
-
"outcome" => "eventOutcome",
|
137
|
-
"proto" => "transportProtocol",
|
138
|
-
"request" => "requestUrl",
|
139
|
-
"rt" => "deviceReceiptTime",
|
140
|
-
"shost" => "sourceHostName",
|
141
|
-
"smac" => "sourceMacAddress",
|
142
|
-
"sntdom" => "sourceNtDomain",
|
143
|
-
"spid" => "sourceProcessId",
|
144
|
-
"spriv" => "sourceUserPrivileges",
|
145
|
-
"sproc" => "sourceProcessName",
|
146
|
-
"spt" => "sourcePort",
|
147
|
-
"src" => "sourceAddress",
|
148
|
-
"start" => "startTime",
|
149
|
-
"suid" => "sourceUserId",
|
150
|
-
"suser" => "sourceUserName",
|
151
|
-
"ahost" => "agentHostName",
|
152
|
-
"art" => "agentReceiptTime",
|
153
|
-
"at" => "agentType",
|
154
|
-
"aid" => "agentId",
|
155
|
-
"_cefVer" => "cefVersion",
|
156
|
-
"agt" => "agentAddress",
|
157
|
-
"av" => "agentVersion",
|
158
|
-
"atz" => "agentTimeZone",
|
159
|
-
"dtz" => "destinationTimeZone",
|
160
|
-
"slong" => "sourceLongitude",
|
161
|
-
"slat" => "sourceLatitude",
|
162
|
-
"dlong" => "destinationLongitude",
|
163
|
-
"dlat" => "destinationLatitude",
|
164
|
-
"catdt" => "categoryDeviceType",
|
165
|
-
"mrt" => "managerReceiptTime",
|
166
|
-
"amac" => "agentMacAddress"
|
167
|
-
}
|
168
|
-
|
169
|
-
# Reverse mapping of CEF full field names to CEF extensions field names for encoding into a CEF event for output.
|
170
|
-
REVERSE_MAPPINGS = MAPPINGS.invert
|
94
|
+
# Defines whether a set of device-specific CEF fields represent the _observer_,
|
95
|
+
# or the actual `host` on which the event occurred. If this codec handles a mix,
|
96
|
+
# it is safe to use the default `observer`.
|
97
|
+
config :device, :validate => %w(observer host), :default => 'observer'
|
171
98
|
|
172
99
|
# A CEF Header is a sequence of zero or more:
|
173
100
|
# - backslash-escaped pipes; OR
|
@@ -255,6 +182,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
255
182
|
@delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n")
|
256
183
|
@buffer = FileWatch::BufferedTokenizer.new(@delimiter)
|
257
184
|
end
|
185
|
+
|
186
|
+
require_relative 'cef/timestamp_normalizer'
|
187
|
+
@timestamp_normalzer = TimestampNormalizer.new(locale: @locale, timezone: @default_timezone)
|
188
|
+
|
189
|
+
generate_header_fields!
|
190
|
+
generate_mappings!
|
258
191
|
end
|
259
192
|
|
260
193
|
public
|
@@ -270,7 +203,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
270
203
|
|
271
204
|
def handle(data, &block)
|
272
205
|
original_data = data.dup
|
273
|
-
event =
|
206
|
+
event = event_factory.new_event
|
274
207
|
event.set(raw_data_field, data) unless raw_data_field.nil?
|
275
208
|
|
276
209
|
@utf8_charset.convert(data)
|
@@ -286,7 +219,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
286
219
|
|
287
220
|
# Use a scanning parser to capture the HEADER_FIELDS
|
288
221
|
unprocessed_data = data
|
289
|
-
|
222
|
+
@header_fields.each do |field_name|
|
290
223
|
match_data = HEADER_SCANNER.match(unprocessed_data)
|
291
224
|
break if match_data.nil? # missing fields
|
292
225
|
|
@@ -304,22 +237,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
304
237
|
message = unprocessed_data
|
305
238
|
|
306
239
|
# Try and parse out the syslog header if there is one
|
307
|
-
|
240
|
+
cef_version_field = @header_fields[0]
|
241
|
+
if (cef_version = event.get(cef_version_field)).include?(' ')
|
308
242
|
split_cef_version = cef_version.rpartition(' ')
|
309
|
-
event.set(
|
310
|
-
event.set(
|
243
|
+
event.set(@syslog_header, split_cef_version[0])
|
244
|
+
event.set(cef_version_field, split_cef_version[2])
|
311
245
|
end
|
312
246
|
|
313
247
|
# Get rid of the CEF bit in the version
|
314
|
-
event.set(
|
248
|
+
event.set(cef_version_field, delete_cef_prefix(event.get(cef_version_field)))
|
315
249
|
|
316
250
|
# Use a scanning parser to capture the Extension Key/Value Pairs
|
317
251
|
if message && message.include?('=')
|
318
252
|
message = message.strip
|
253
|
+
extension_fields = {}
|
319
254
|
|
320
255
|
message.scan(EXTENSION_KEY_VALUE_SCANNER) do |extension_field_key, raw_extension_field_value|
|
321
256
|
# expand abbreviated extension field keys
|
322
|
-
extension_field_key =
|
257
|
+
extension_field_key = @decode_mapping.fetch(extension_field_key, extension_field_key)
|
323
258
|
|
324
259
|
# convert extension field name to strict legal field_reference, fixing field names with ambiguous array-like syntax
|
325
260
|
extension_field_key = extension_field_key.sub(EXTENSION_KEY_ARRAY_CAPTURE, '[\1]\2') if extension_field_key.end_with?(']')
|
@@ -327,7 +262,21 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
327
262
|
# process legal extension field value escapes
|
328
263
|
extension_field_value = raw_extension_field_value.gsub(EXTENSION_VALUE_ESCAPE_CAPTURE, '\1')
|
329
264
|
|
330
|
-
|
265
|
+
extension_fields[extension_field_key] = extension_field_value
|
266
|
+
end
|
267
|
+
|
268
|
+
# in ECS mode, normalize timestamps including timezone.
|
269
|
+
if ecs_compatibility != :disabled
|
270
|
+
device_timezone = extension_fields['[event][timezone]']
|
271
|
+
@timestamp_fields.each do |timestamp_field_name|
|
272
|
+
raw_timestamp = extension_fields.delete(timestamp_field_name) or next
|
273
|
+
value = normalize_timestamp(raw_timestamp, device_timezone)
|
274
|
+
event.set(timestamp_field_name, value)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
extension_fields.each do |field_key, field_value|
|
279
|
+
event.set(field_key, field_value)
|
331
280
|
end
|
332
281
|
end
|
333
282
|
|
@@ -335,7 +284,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
335
284
|
rescue => e
|
336
285
|
@logger.error("Failed to decode CEF payload. Generating failure event with payload in message field.",
|
337
286
|
:exception => e.class, :message => e.message, :backtrace => e.backtrace, :original_data => original_data)
|
338
|
-
yield
|
287
|
+
yield event_factory.new_event("message" => data, "tags" => ["_cefparsefailure"])
|
339
288
|
end
|
340
289
|
|
341
290
|
public
|
@@ -368,6 +317,212 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
368
317
|
|
369
318
|
private
|
370
319
|
|
320
|
+
def generate_header_fields!
|
321
|
+
# @header_fields is an _ordered_ set of fields.
|
322
|
+
@header_fields = [
|
323
|
+
ecs_select[disabled: 'cefVersion', v1: '[cef][version]'],
|
324
|
+
ecs_select[disabled: 'deviceVendor', v1: '[observer][vendor]'],
|
325
|
+
ecs_select[disabled: 'deviceProduct', v1: '[observer][product]'],
|
326
|
+
ecs_select[disabled: 'deviceVersion', v1: '[observer][version]'],
|
327
|
+
ecs_select[disabled: 'deviceEventClassId', v1: '[event][code]'],
|
328
|
+
ecs_select[disabled: 'name', v1: '[cef][name]'],
|
329
|
+
ecs_select[disabled: 'severity', v1: '[event][severity]']
|
330
|
+
].map(&:freeze).freeze
|
331
|
+
# the @syslog_header is the field name used when a syslog header preceeds the CEF Version.
|
332
|
+
@syslog_header = ecs_select[disabled:'syslog',v1:'[log][syslog][header]']
|
333
|
+
end
|
334
|
+
|
335
|
+
class CEFField
|
336
|
+
##
|
337
|
+
# @param name [String]: the full CEF name of a field
|
338
|
+
# @param key [String] (optional): an abbreviated CEF key to use when encoding a value with `reverse_mapping => true`
|
339
|
+
# when left unspecified, the `key` is the field's `name`.
|
340
|
+
# @param ecs_field [String] (optional): an ECS-compatible field reference to use, with square-bracket syntax.
|
341
|
+
# when left unspecified, the `ecs_field` is the field's `name`.
|
342
|
+
# @param legacy [String] (optional): a legacy CEF name to support in pass-through.
|
343
|
+
# in decoding mode without ECS, field name will be used as-provided.
|
344
|
+
# in encoding mode without ECS when provided to `fields` and `reverse_mapping => false`,
|
345
|
+
# field name will be used as-provided.
|
346
|
+
# @param priority [Integer] (optional): when multiple fields resolve to the same ECS field name, the field with the
|
347
|
+
# highest `prioriry` will be used by the encoder.
|
348
|
+
def initialize(name, key: name, ecs_field: name, legacy:nil, priority:0, normalize:nil)
|
349
|
+
@name = name
|
350
|
+
@key = key
|
351
|
+
@ecs_field = ecs_field
|
352
|
+
@legacy = legacy
|
353
|
+
@priority = priority
|
354
|
+
@normalize = normalize
|
355
|
+
end
|
356
|
+
attr_reader :name
|
357
|
+
attr_reader :key
|
358
|
+
attr_reader :ecs_field
|
359
|
+
attr_reader :legacy
|
360
|
+
attr_reader :priority
|
361
|
+
attr_reader :normalize
|
362
|
+
end
|
363
|
+
|
364
|
+
def generate_mappings!
|
365
|
+
encode_mapping = Hash.new
|
366
|
+
decode_mapping = Hash.new
|
367
|
+
timestamp_fields = Set.new
|
368
|
+
[
|
369
|
+
CEFField.new("agentAddress", key: "agt", ecs_field: "[agent][ip]"),
|
370
|
+
CEFField.new("agentDnsDomain", ecs_field: "[cef][agent][registered_domain]", priority: 10),
|
371
|
+
CEFField.new("agentHostName", key: "ahost", ecs_field: "[agent][name]"),
|
372
|
+
CEFField.new("agentId", key: "aid", ecs_field: "[agent][id]"),
|
373
|
+
CEFField.new("agentMacAddress", key: "amac", ecs_field: "[agent][mac]"),
|
374
|
+
CEFField.new("agentNtDomain", ecs_field: "[cef][agent][registered_domain]"),
|
375
|
+
CEFField.new("agentReceiptTime", key: "art", ecs_field: "[event][created]", normalize: :timestamp),
|
376
|
+
CEFField.new("agentTimeZone", key: "atz", ecs_field: "[cef][agent][timezone]"),
|
377
|
+
CEFField.new("agentTranslatedAddress", ecs_field: "[cef][agent][nat][ip]"),
|
378
|
+
CEFField.new("agentTranslatedZoneExternalID", ecs_field: "[cef][agent][translated_zone][external_id]"),
|
379
|
+
CEFField.new("agentTranslatedZoneURI", ecs_field: "[cef][agent][translated_zone][uri]"),
|
380
|
+
CEFField.new("agentType", key: "at", ecs_field: "[agent][type]"),
|
381
|
+
CEFField.new("agentVersion", key: "av", ecs_field: "[agent][version]"),
|
382
|
+
CEFField.new("agentZoneExternalID", ecs_field: "[cef][agent][zone][external_id]"),
|
383
|
+
CEFField.new("agentZoneURI", ecs_field: "[cef][agent][zone][uri]"),
|
384
|
+
CEFField.new("applicationProtocol", key: "app", ecs_field: "[network][protocol]"),
|
385
|
+
CEFField.new("baseEventCount", key: "cnt", ecs_field: "[cef][base_event_count]"),
|
386
|
+
CEFField.new("bytesIn", key: "in", ecs_field: "[source][bytes]"),
|
387
|
+
CEFField.new("bytesOut", key: "out", ecs_field: "[destination][bytes]"),
|
388
|
+
CEFField.new("categoryDeviceType", key: "catdt", ecs_field: "[cef][device_type]"),
|
389
|
+
CEFField.new("customerExternalID", ecs_field: "[organization][id]"),
|
390
|
+
CEFField.new("customerURI", ecs_field: "[organization][name]"),
|
391
|
+
CEFField.new("destinationAddress", key: "dst", ecs_field: "[destination][ip]"),
|
392
|
+
CEFField.new("destinationDnsDomain", ecs_field: "[destination][registered_domain]", priority: 10),
|
393
|
+
CEFField.new("destinationGeoLatitude", key: "dlat", ecs_field: "[destination][geo][location][lat]", legacy: "destinationLatitude"),
|
394
|
+
CEFField.new("destinationGeoLongitude", key: "dlong", ecs_field: "[destination][geo][location][lon]", legacy: "destinationLongitude"),
|
395
|
+
CEFField.new("destinationHostName", key: "dhost", ecs_field: "[destination][domain]"),
|
396
|
+
CEFField.new("destinationMacAddress", key: "dmac", ecs_field: "[destination][mac]"),
|
397
|
+
CEFField.new("destinationNtDomain", key: "dntdom", ecs_field: "[destination][registered_domain]"),
|
398
|
+
CEFField.new("destinationPort", key: "dpt", ecs_field: "[destination][port]"),
|
399
|
+
CEFField.new("destinationProcessId", key: "dpid", ecs_field: "[destination][process][pid]"),
|
400
|
+
CEFField.new("destinationProcessName", key: "dproc", ecs_field: "[destination][process][name]"),
|
401
|
+
CEFField.new("destinationServiceName", ecs_field: "[destination][service][name]"),
|
402
|
+
CEFField.new("destinationTranslatedAddress", ecs_field: "[destination][nat][ip]"),
|
403
|
+
CEFField.new("destinationTranslatedPort", ecs_field: "[destination][nat][port]"),
|
404
|
+
CEFField.new("destinationTranslatedZoneExternalID", ecs_field: "[cef][destination][translated_zone][external_id]"),
|
405
|
+
CEFField.new("destinationTranslatedZoneURI", ecs_field: "[cef][destination][translated_zone][uri]"),
|
406
|
+
CEFField.new("destinationUserId", key: "duid", ecs_field: "[destination][user][id]"),
|
407
|
+
CEFField.new("destinationUserName", key: "duser", ecs_field: "[destination][user][name]"),
|
408
|
+
CEFField.new("destinationUserPrivileges", key: "dpriv", ecs_field: "[destination][user][group][name]"),
|
409
|
+
CEFField.new("destinationZoneExternalID", ecs_field: "[cef][destination][zone][external_id]"),
|
410
|
+
CEFField.new("destinationZoneURI", ecs_field: "[cef][destination][zone][uri]"),
|
411
|
+
CEFField.new("deviceAction", key: "act", ecs_field: "[event][action]"),
|
412
|
+
CEFField.new("deviceAddress", key: "dvc", ecs_field: "[#{@device}][ip]"),
|
413
|
+
(1..15).map do |idx|
|
414
|
+
[
|
415
|
+
CEFField.new("deviceCustomFloatingPoint#{idx}", key: "cfp#{idx}", ecs_field: "[cef][device_custom_floating_point_#{idx}][value]"),
|
416
|
+
CEFField.new("deviceCustomFloatingPoint#{idx}Label", key: "cfp#{idx}Label", ecs_field: "[cef][device_custom_floating_point_#{idx}][label]"),
|
417
|
+
CEFField.new("deviceCustomIPv6Address#{idx}", key: "c6a#{idx}", ecs_field: "[cef][device_custom_ipv6_address_#{idx}][value]"),
|
418
|
+
CEFField.new("deviceCustomIPv6Address#{idx}Label", key: "c6a#{idx}Label", ecs_field: "[cef][device_custom_ipv6_address_#{idx}][label]"),
|
419
|
+
CEFField.new("deviceCustomNumber#{idx}", key: "cn#{idx}", ecs_field: "[cef][device_custom_number_#{idx}][value]"),
|
420
|
+
CEFField.new("deviceCustomNumber#{idx}Label", key: "cn#{idx}Label", ecs_field: "[cef][device_custom_number_#{idx}][label]"),
|
421
|
+
CEFField.new("deviceCustomString#{idx}", key: "cs#{idx}", ecs_field: "[cef][device_custom_string_#{idx}][value]"),
|
422
|
+
CEFField.new("deviceCustomString#{idx}Label", key: "cs#{idx}Label", ecs_field: "[cef][device_custom_string_#{idx}][label]"),
|
423
|
+
]
|
424
|
+
end,
|
425
|
+
CEFField.new("deviceDirection", ecs_field: "[network][direction]"),
|
426
|
+
CEFField.new("deviceDnsDomain", ecs_field: "[#{@device}][registered_domain]", priority: 10),
|
427
|
+
CEFField.new("deviceEventCategory", key: "cat", ecs_field: "[cef][category]"),
|
428
|
+
CEFField.new("deviceExternalId", ecs_field: (@device == 'host' ? "[host][id]" : "[observer][name]")),
|
429
|
+
CEFField.new("deviceFacility", ecs_field: "[log][syslog][facility][code]"),
|
430
|
+
CEFField.new("deviceHostName", key: "dvchost", ecs_field: (@device == 'host' ? '[host][name]' : '[observer][hostname]')),
|
431
|
+
CEFField.new("deviceInboundInterface", ecs_field: "[observer][ingress][interface][name]"),
|
432
|
+
CEFField.new("deviceMacAddress", key: "dvcmac", ecs_field: "[#{@device}][mac]"),
|
433
|
+
CEFField.new("deviceNtDomain", ecs_field: "[cef][nt_domain]"),
|
434
|
+
CEFField.new("deviceOutboundInterface", ecs_field: "[observer][egress][interface][name]"),
|
435
|
+
CEFField.new("devicePayloadId", ecs_field: "[cef][payload_id]"),
|
436
|
+
CEFField.new("deviceProcessId", key: "dvcpid", ecs_field: "[process][pid]"),
|
437
|
+
CEFField.new("deviceProcessName", ecs_field: "[process][name]"),
|
438
|
+
CEFField.new("deviceReceiptTime", key: "rt", ecs_field: "@timestamp", normalize: :timestamp),
|
439
|
+
CEFField.new("deviceTimeZone", key: "dtz", ecs_field: "[event][timezone]", legacy: "destinationTimeZone"),
|
440
|
+
CEFField.new("deviceTranslatedAddress", ecs_field: "[host][nat][ip]"),
|
441
|
+
CEFField.new("deviceTranslatedZoneExternalID", ecs_field: "[cef][translated_zone][external_id]"),
|
442
|
+
CEFField.new("deviceTranslatedZoneURI", ecs_field: "[cef][translated_zone][uri]"),
|
443
|
+
CEFField.new("deviceVersion", ecs_field: "[observer][version]"),
|
444
|
+
CEFField.new("deviceZoneExternalID", ecs_field: "[cef][zone][external_id]"),
|
445
|
+
CEFField.new("deviceZoneURI", ecs_field: "[cef][zone][uri]"),
|
446
|
+
CEFField.new("endTime", key: "end", ecs_field: "[event][end]", normalize: :timestamp),
|
447
|
+
CEFField.new("eventId", ecs_field: "[event][id]"),
|
448
|
+
CEFField.new("eventOutcome", key: "outcome", ecs_field: "[event][outcome]"),
|
449
|
+
CEFField.new("externalId", ecs_field: "[cef][external_id]"),
|
450
|
+
CEFField.new("fileCreateTime", ecs_field: "[file][created]"),
|
451
|
+
CEFField.new("fileHash", ecs_field: "[file][hash]"),
|
452
|
+
CEFField.new("fileId", ecs_field: "[file][inode]"),
|
453
|
+
CEFField.new("fileModificationTime", ecs_field: "[file][mtime]", normalize: :timestamp),
|
454
|
+
CEFField.new("fileName", key: "fname", ecs_field: "[file][name]"),
|
455
|
+
CEFField.new("filePath", ecs_field: "[file][path]"),
|
456
|
+
CEFField.new("filePermission", ecs_field: "[file][group]"),
|
457
|
+
CEFField.new("fileSize", key: "fsize", ecs_field: "[file][size]"),
|
458
|
+
CEFField.new("fileType", ecs_field: "[file][extension]"),
|
459
|
+
CEFField.new("managerReceiptTime", key: "mrt", ecs_field: "[event][ingested]", normalize: :timestamp),
|
460
|
+
CEFField.new("message", key: "msg", ecs_field: "[message]"),
|
461
|
+
CEFField.new("oldFileCreateTime", ecs_field: "[cef][old_file][created]", normalize: :timestamp),
|
462
|
+
CEFField.new("oldFileHash", ecs_field: "[cef][old_file][hash]"),
|
463
|
+
CEFField.new("oldFileId", ecs_field: "[cef][old_file][inode]"),
|
464
|
+
CEFField.new("oldFileModificationTime", ecs_field: "[cef][old_file][mtime]", normalize: :timestamp),
|
465
|
+
CEFField.new("oldFileName", ecs_field: "[cef][old_file][name]"),
|
466
|
+
CEFField.new("oldFilePath", ecs_field: "[cef][old_file][path]"),
|
467
|
+
CEFField.new("oldFilePermission", ecs_field: "[cef][old_file][group]"),
|
468
|
+
CEFField.new("oldFileSize", ecs_field: "[cef][old_file][size]"),
|
469
|
+
CEFField.new("oldFileType", ecs_field: "[cef][old_file][extension]"),
|
470
|
+
CEFField.new("rawEvent", ecs_field: "[event][original]"),
|
471
|
+
CEFField.new("Reason", key: "reason", ecs_field: "[event][reason]"),
|
472
|
+
CEFField.new("requestClientApplication", ecs_field: "[user_agent][original]"),
|
473
|
+
CEFField.new("requestContext", ecs_field: "[http][request][referrer]"),
|
474
|
+
CEFField.new("requestCookies", ecs_field: "[cef][request][cookies]"),
|
475
|
+
CEFField.new("requestMethod", ecs_field: "[http][request][method]"),
|
476
|
+
CEFField.new("requestUrl", key: "request", ecs_field: "[url][original]"),
|
477
|
+
CEFField.new("sourceAddress", key: "src", ecs_field: "[source][ip]"),
|
478
|
+
CEFField.new("sourceDnsDomain", ecs_field: "[source][registered_domain]", priority: 10),
|
479
|
+
CEFField.new("sourceGeoLatitude", key: "slat", ecs_field: "[source][geo][location][lat]", legacy: "sourceLatitude"),
|
480
|
+
CEFField.new("sourceGeoLongitude", key: "slong", ecs_field: "[source][geo][location][lon]", legacy: "sourceLongitude"),
|
481
|
+
CEFField.new("sourceHostName", key: "shost", ecs_field: "[source][domain]"),
|
482
|
+
CEFField.new("sourceMacAddress", key: "smac", ecs_field: "[source][mac]"),
|
483
|
+
CEFField.new("sourceNtDomain", key: "sntdom", ecs_field: "[source][registered_domain]"),
|
484
|
+
CEFField.new("sourcePort", key: "spt", ecs_field: "[source][port]"),
|
485
|
+
CEFField.new("sourceProcessId", key: "spid", ecs_field: "[source][process][pid]"),
|
486
|
+
CEFField.new("sourceProcessName", key: "sproc", ecs_field: "[source][process][name]"),
|
487
|
+
CEFField.new("sourceServiceName", ecs_field: "[source][service][name]"),
|
488
|
+
CEFField.new("sourceTranslatedAddress", ecs_field: "[source][nat][ip]"),
|
489
|
+
CEFField.new("sourceTranslatedPort", ecs_field: "[source][nat][port]"),
|
490
|
+
CEFField.new("sourceTranslatedZoneExternalID", ecs_field: "[cef][source][translated_zone][external_id]"),
|
491
|
+
CEFField.new("sourceTranslatedZoneURI", ecs_field: "[cef][source][translated_zone][uri]"),
|
492
|
+
CEFField.new("sourceUserId", key: "suid", ecs_field: "[source][user][id]"),
|
493
|
+
CEFField.new("sourceUserName", key: "suser", ecs_field: "[source][user][name]"),
|
494
|
+
CEFField.new("sourceUserPrivileges", key: "spriv", ecs_field: "[source][user][group][name]"),
|
495
|
+
CEFField.new("sourceZoneExternalID", ecs_field: "[cef][source][zone][external_id]"),
|
496
|
+
CEFField.new("sourceZoneURI", ecs_field: "[cef][source][zone][uri]"),
|
497
|
+
CEFField.new("startTime", key: "start", ecs_field: "[event][start]", normalize: :timestamp),
|
498
|
+
CEFField.new("transportProtocol", key: "proto", ecs_field: "[network][transport]"),
|
499
|
+
CEFField.new("type", ecs_field: "[cef][type]"),
|
500
|
+
].flatten.sort_by(&:priority).each do |cef|
|
501
|
+
field_name = ecs_select[disabled:cef.name, v1:cef.ecs_field]
|
502
|
+
|
503
|
+
# whether the source is a cef_key or cef_name, normalize to field_name
|
504
|
+
decode_mapping[cef.key] = field_name
|
505
|
+
decode_mapping[cef.name] = field_name
|
506
|
+
|
507
|
+
# whether source is a cef_name or a field_name, normalize to target
|
508
|
+
normalized_encode_target = @reverse_mapping ? cef.key : cef.name
|
509
|
+
encode_mapping[field_name] = normalized_encode_target
|
510
|
+
encode_mapping[cef.name] = normalized_encode_target unless cef.name == field_name
|
511
|
+
|
512
|
+
# if a field has an alias, normalize pass-through
|
513
|
+
if cef.legacy
|
514
|
+
decode_mapping[cef.legacy] = ecs_select[disabled:cef.legacy, v1:cef.ecs_field]
|
515
|
+
encode_mapping[cef.legacy] = @reverse_mapping ? cef.key : cef.legacy
|
516
|
+
end
|
517
|
+
|
518
|
+
timestamp_fields << field_name if ecs_compatibility != :disabled && cef.normalize == :timestamp
|
519
|
+
end
|
520
|
+
|
521
|
+
@decode_mapping = decode_mapping.dup.freeze
|
522
|
+
@encode_mapping = encode_mapping.dup.freeze
|
523
|
+
@timestamp_fields = timestamp_fields.dup.freeze
|
524
|
+
end
|
525
|
+
|
371
526
|
# Escape pipes and backslashes in the header. Equal signs are ok.
|
372
527
|
# Newlines are forbidden.
|
373
528
|
def sanitize_header_field(value)
|
@@ -392,17 +547,23 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
392
547
|
.gsub(EXTENSION_VALUE_SANITIZER_PATTERN, EXTENSION_VALUE_SANITIZER_MAPPING)
|
393
548
|
end
|
394
549
|
|
550
|
+
def normalize_timestamp(value, device_timezone_name)
|
551
|
+
value = @timestamp_normalzer.normalize(value, device_timezone_name).iso8601(9)
|
552
|
+
|
553
|
+
LogStash::Timestamp.new(value)
|
554
|
+
rescue => e
|
555
|
+
@logger.error("Failed to parse CEF timestamp value `#{value}` (#{e.message})")
|
556
|
+
raise InvalidTimestamp.new("Not a valid CEF timestamp: `#{value}`")
|
557
|
+
end
|
558
|
+
|
395
559
|
def get_value(fieldname, event)
|
396
560
|
val = event.get(fieldname)
|
397
561
|
|
398
562
|
return nil if val.nil?
|
399
563
|
|
400
|
-
key =
|
401
|
-
|
402
|
-
|
403
|
-
key = REVERSE_MAPPINGS[key] || key
|
404
|
-
end
|
405
|
-
|
564
|
+
key = @encode_mapping.fetch(fieldname, fieldname)
|
565
|
+
key = sanitize_extension_key(key)
|
566
|
+
|
406
567
|
case val
|
407
568
|
when Array, Hash
|
408
569
|
return "#{key}=#{sanitize_extension_val(val.to_json)}"
|