logstash-codec-cef 6.1.2-java → 6.2.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +122 -8
- data/lib/logstash/codecs/cef.rb +290 -109
- data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
- data/logstash-codec-cef.gemspec +2 -1
- data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
- data/spec/codecs/cef_spec.rb +477 -306
- metadata +19 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 25264d450cdfa027ac9758a05972c0c87119a635238b9bacfc746e4daec40dff
|
|
4
|
+
data.tar.gz: 63dbd8558c231e61c9e55962484042a52c28177cba5b4adcf4d49291aace491a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 62ac45d798abaf3008f99357b578506237bddc024904951cab64bdb36fd61f1c91894e4e70f2ce6a911fde46c4d1a15c69a1368b2a0dd69a6f509e09e8ee8192
|
|
7
|
+
data.tar.gz: 46dd75f39b72ae788dfcd29f21810be7086bb16cb1984f81f50a786682d66f06f7e6a2ee543f3e88a3e8951da5680f99fdcd45d28e606c16171d34bc30ceb4a4
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
## 6.2.0
|
|
2
|
+
- Introduce ECS Compatibility mode [#83](https://github.com/logstash-plugins/logstash-codec-cef/pull/83).
|
|
3
|
+
|
|
1
4
|
## 6.1.2
|
|
2
5
|
- Added error log with full payload when something bad happens in decoding a message[#84](https://github.com/logstash-plugins/logstash-codec-cef/pull/84)
|
|
3
6
|
|
data/docs/index.asciidoc
CHANGED
|
@@ -27,14 +27,53 @@ https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-docum
|
|
|
27
27
|
If this codec receives a payload from an input that is not a valid CEF message, then it will
|
|
28
28
|
produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
|
29
29
|
|
|
30
|
+
==== Compatibility with the Elastic Common Schema (ECS)
|
|
31
|
+
|
|
32
|
+
This plugin can be used to decode CEF events _into_ the Elastic Common Schema, or to encode ECS-compatible events into CEF.
|
|
33
|
+
It can also be used _without_ ECS, encoding and decoding events using only CEF-defined field names and keys.
|
|
34
|
+
|
|
35
|
+
The ECS Compatibility mode for a specific plugin instance can be controlled by setting <<plugins-{type}s-{plugin}-ecs_compatibility>> when defining the codec:
|
|
36
|
+
|
|
37
|
+
[source,sh]
|
|
38
|
+
-----
|
|
39
|
+
input {
|
|
40
|
+
tcp {
|
|
41
|
+
# ...
|
|
42
|
+
codec => cef {
|
|
43
|
+
ecs_compatibility => v1
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
-----
|
|
48
|
+
|
|
49
|
+
If left unspecified, the value of the `pipeline.ecs_compatibility` setting is used.
|
|
50
|
+
|
|
51
|
+
===== Timestamps and ECS Compatiblity
|
|
52
|
+
|
|
53
|
+
When running in ECS Compatibility Mode, timestamp-type fields are parsed and normalized
|
|
54
|
+
to specific points on the timeline.
|
|
55
|
+
|
|
56
|
+
Because the CEF format allows ambiguous timestamp formats, some reasonable assumptions are made:
|
|
57
|
+
|
|
58
|
+
- When the timestamp does not include a year, we assume it happened in the recent past
|
|
59
|
+
(or _very_ near future to accommodate out-of-sync clocks and timezone offsets).
|
|
60
|
+
- When the timestamp does not include UTC-offset information, we use the event's
|
|
61
|
+
timezone (`dtz` or `deviceTimeZone` field), or fall through to this plugin's
|
|
62
|
+
<<plugins-{type}s-{plugin}-default_timezone>>.
|
|
63
|
+
- Localized timestamps are parsed using the provided <<plugins-{type}s-{plugin}-locale>>.
|
|
64
|
+
|
|
30
65
|
[id="plugins-{type}s-{plugin}-options"]
|
|
31
66
|
==== Cef Codec Configuration Options
|
|
32
67
|
|
|
33
68
|
[cols="<,<,<",options="header",]
|
|
34
69
|
|=======================================================================
|
|
35
70
|
|Setting |Input type|Required
|
|
71
|
+
| <<plugins-{type}s-{plugin}-default_timezone>> |<<string,string>>|No
|
|
36
72
|
| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
|
|
73
|
+
| <<plugins-{type}s-{plugin}-device>> |<<string,string>>|No
|
|
74
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
|
|
37
75
|
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
|
76
|
+
| <<plugins-{type}s-{plugin}-locale>> |<<string,string>>|No
|
|
38
77
|
| <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
|
|
39
78
|
| <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
|
|
40
79
|
| <<plugins-{type}s-{plugin}-reverse_mapping>> |<<boolean,boolean>>|No
|
|
@@ -46,6 +85,21 @@ produce an event with the payload as the 'message' field and a '_cefparsefailure
|
|
|
46
85
|
|
|
47
86
|
|
|
48
87
|
|
|
88
|
+
[id="plugins-{type}s-{plugin}-default_timezone"]
|
|
89
|
+
===== `default_timezone`
|
|
90
|
+
|
|
91
|
+
* Value type is <<string,string>>
|
|
92
|
+
* Supported values are:
|
|
93
|
+
** https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[Timezone names] (such as `Europe/Moscow`, `America/Argentina/Buenos_Aires`)
|
|
94
|
+
** UTC Offsets (such as `-08:00`, `+03:00`)
|
|
95
|
+
* The default value is your system time zone
|
|
96
|
+
* This option has no effect when _encoding_.
|
|
97
|
+
|
|
98
|
+
When parsing timestamp fields in ECS mode and encountering timestamps that
|
|
99
|
+
do not contain UTC-offset information, the `deviceTimeZone` (`dtz`) field
|
|
100
|
+
from the CEF payload is used to interpret the given time. If the event does
|
|
101
|
+
not include timezone information, this `default_timezone` is used instead.
|
|
102
|
+
|
|
49
103
|
[id="plugins-{type}s-{plugin}-delimiter"]
|
|
50
104
|
===== `delimiter`
|
|
51
105
|
|
|
@@ -69,21 +123,71 @@ This setting allows the following character sequences to have special meaning:
|
|
|
69
123
|
* `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
|
|
70
124
|
* `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
|
71
125
|
|
|
126
|
+
[id="plugins-{type}s-{plugin}-device"]
|
|
127
|
+
===== `device`
|
|
128
|
+
|
|
129
|
+
* Value type is <<string,string>>
|
|
130
|
+
* Supported values are:
|
|
131
|
+
** `observer`: indicates that device-specific fields represent the device used to _observe_ the event.
|
|
132
|
+
** `host`: indicates that device-specific fields represent the device on which the event _occurred_.
|
|
133
|
+
* The default value for this setting is `observer`.
|
|
134
|
+
* Option has no effect when <<plugins-{type}s-{plugin}-ecs_compatibility,`ecs_compatibility => disabled`>>.
|
|
135
|
+
* Option has no effect when _encoding_
|
|
136
|
+
|
|
137
|
+
Defines a set of device-specific CEF fields as either representing the device on which an
|
|
138
|
+
event _occurred_, or merely the device from which the event was _observed_.
|
|
139
|
+
This causes the relevant fields to be routed to either the `host` or the `observer`
|
|
140
|
+
top-level groupings.
|
|
141
|
+
|
|
142
|
+
If the codec handles data from a variety of sources, the ECS recommendation is to use `observer`.
|
|
143
|
+
|
|
144
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
|
145
|
+
===== `ecs_compatibility`
|
|
146
|
+
|
|
147
|
+
* Value type is <<string,string>>
|
|
148
|
+
* Supported values are:
|
|
149
|
+
** `disabled`: uses CEF-defined field names in the event (e.g., `bytesIn`, `sourceAddress`)
|
|
150
|
+
** `v1`: supports ECS-compatible event fields (e.g., `[source][bytes]`, `[source][ip]`)
|
|
151
|
+
* Default value depends on which version of Logstash is running:
|
|
152
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
|
153
|
+
** Otherwise, the default value is `disabled`.
|
|
154
|
+
|
|
155
|
+
Controls this plugin's compatibility with the
|
|
156
|
+
{ecs-ref}[Elastic Common Schema (ECS)]
|
|
157
|
+
(ECS)].
|
|
158
|
+
|
|
72
159
|
[id="plugins-{type}s-{plugin}-fields"]
|
|
73
160
|
===== `fields`
|
|
74
161
|
|
|
75
162
|
* Value type is <<array,array>>
|
|
76
163
|
* Default value is `[]`
|
|
164
|
+
* Option has no effect when _decoding_
|
|
165
|
+
|
|
166
|
+
When this codec is used in an Output Plugin, a list of fields can be provided to be included in CEF extensions part as key/value pairs.
|
|
77
167
|
|
|
78
|
-
|
|
168
|
+
[id="plugins-{type}s-{plugin}-locale"]
|
|
169
|
+
===== `locale`
|
|
170
|
+
|
|
171
|
+
* Value type is <<string,string>>
|
|
172
|
+
* Supported values are:
|
|
173
|
+
** Abbreviated language_COUNTRY format (e.g., `en_GB`, `pt_BR`)
|
|
174
|
+
** Valid https://tools.ietf.org/html/bcp47[IETF BCP 47] language tag (e.g., `zh-cmn-Hans-CN`)
|
|
175
|
+
* The default value is your system locale
|
|
176
|
+
* Option has no effect when _encoding_
|
|
177
|
+
|
|
178
|
+
When parsing timestamp fields in ECS mode and encountering timestamps in
|
|
179
|
+
a localized format, this `locale` is used to interpret locale-specific strings
|
|
180
|
+
such as month abbreviations.
|
|
79
181
|
|
|
80
182
|
[id="plugins-{type}s-{plugin}-name"]
|
|
81
183
|
===== `name`
|
|
82
184
|
|
|
83
185
|
* Value type is <<string,string>>
|
|
84
186
|
* Default value is `"Logstash"`
|
|
187
|
+
* Option has no effect when _decoding_
|
|
85
188
|
|
|
86
|
-
|
|
189
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
190
|
+
value of the name field in the CEF header. The new value can include `%{foo}` strings
|
|
87
191
|
to help you build a new value from other parts of the event.
|
|
88
192
|
|
|
89
193
|
[id="plugins-{type}s-{plugin}-product"]
|
|
@@ -91,8 +195,10 @@ to help you build a new value from other parts of the event.
|
|
|
91
195
|
|
|
92
196
|
* Value type is <<string,string>>
|
|
93
197
|
* Default value is `"Logstash"`
|
|
198
|
+
* Option has no effect when _decoding_
|
|
94
199
|
|
|
95
|
-
|
|
200
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
201
|
+
value of the device product field in CEF header. The new value can include `%{foo}` strings
|
|
96
202
|
to help you build a new value from other parts of the event.
|
|
97
203
|
|
|
98
204
|
|
|
@@ -101,6 +207,7 @@ to help you build a new value from other parts of the event.
|
|
|
101
207
|
|
|
102
208
|
* Value type is <<boolean,boolean>>
|
|
103
209
|
* Default value is `false`
|
|
210
|
+
* Option has no effect when _decoding_
|
|
104
211
|
|
|
105
212
|
Set to true to adhere to the specifications and encode using the CEF key name (short name) for the CEF field names.
|
|
106
213
|
|
|
@@ -109,8 +216,10 @@ Set to true to adhere to the specifications and encode using the CEF key name (s
|
|
|
109
216
|
|
|
110
217
|
* Value type is <<string,string>>
|
|
111
218
|
* Default value is `"6"`
|
|
219
|
+
* Option has no effect when _decoding_
|
|
112
220
|
|
|
113
|
-
|
|
221
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
222
|
+
value of the severity field in CEF header. The new value can include `%{foo}` strings
|
|
114
223
|
to help you build a new value from other parts of the event.
|
|
115
224
|
|
|
116
225
|
Defined as field of type string to allow sprintf. The value will be validated
|
|
@@ -122,8 +231,10 @@ All invalid values will be mapped to the default of 6.
|
|
|
122
231
|
|
|
123
232
|
* Value type is <<string,string>>
|
|
124
233
|
* Default value is `"Logstash"`
|
|
234
|
+
* Option has no effect when _decoding_
|
|
125
235
|
|
|
126
|
-
|
|
236
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
237
|
+
value of the signature ID field in CEF header. The new value can include `%{foo}` strings
|
|
127
238
|
to help you build a new value from other parts of the event.
|
|
128
239
|
|
|
129
240
|
[id="plugins-{type}s-{plugin}-vendor"]
|
|
@@ -131,8 +242,10 @@ to help you build a new value from other parts of the event.
|
|
|
131
242
|
|
|
132
243
|
* Value type is <<string,string>>
|
|
133
244
|
* Default value is `"Elasticsearch"`
|
|
245
|
+
* Option has no effect when _decoding_
|
|
134
246
|
|
|
135
|
-
|
|
247
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
248
|
+
value of the device vendor field in CEF header. The new value can include `%{foo}` strings
|
|
136
249
|
to help you build a new value from other parts of the event.
|
|
137
250
|
|
|
138
251
|
[id="plugins-{type}s-{plugin}-version"]
|
|
@@ -140,8 +253,9 @@ to help you build a new value from other parts of the event.
|
|
|
140
253
|
|
|
141
254
|
* Value type is <<string,string>>
|
|
142
255
|
* Default value is `"1.0"`
|
|
256
|
+
* Option has no effect when _decoding_
|
|
143
257
|
|
|
144
|
-
|
|
258
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
259
|
+
value of the device version field in CEF header. The new value can include `%{foo}` strings
|
|
145
260
|
to help you build a new value from other parts of the event.
|
|
146
261
|
|
|
147
|
-
|
data/lib/logstash/codecs/cef.rb
CHANGED
|
@@ -3,6 +3,9 @@ require "logstash/util/buftok"
|
|
|
3
3
|
require "logstash/util/charset"
|
|
4
4
|
require "logstash/codecs/base"
|
|
5
5
|
require "json"
|
|
6
|
+
require "time"
|
|
7
|
+
|
|
8
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
|
6
9
|
|
|
7
10
|
# Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
|
8
11
|
# Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
|
@@ -13,6 +16,10 @@ require "json"
|
|
|
13
16
|
class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
14
17
|
config_name "cef"
|
|
15
18
|
|
|
19
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
|
|
20
|
+
|
|
21
|
+
InvalidTimestamp = Class.new(StandardError)
|
|
22
|
+
|
|
16
23
|
# Device vendor field in CEF header. The new value can include `%{foo}` strings
|
|
17
24
|
# to help you build a new value from other parts of the event.
|
|
18
25
|
config :vendor, :validate => :string, :default => "Elasticsearch"
|
|
@@ -68,106 +75,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
68
75
|
# * `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
|
69
76
|
config :delimiter, :validate => :string
|
|
70
77
|
|
|
78
|
+
# When parsing timestamps that do not include a UTC offset in payloads that do not
|
|
79
|
+
# include the device's timezone, the default timezone is used.
|
|
80
|
+
# If none is provided the system timezone is used.
|
|
81
|
+
config :default_timezone, :validate => :string
|
|
82
|
+
|
|
83
|
+
# The locale is used to parse abbreviated month names from some CEF timestamp
|
|
84
|
+
# formats.
|
|
85
|
+
# If none is provided, the system default is used.
|
|
86
|
+
config :locale, :validate => :string
|
|
87
|
+
|
|
71
88
|
# If raw_data_field is set, during decode of an event an additional field with
|
|
72
89
|
# the provided name is added, which contains the raw data.
|
|
73
90
|
config :raw_data_field, :validate => :string
|
|
74
91
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
#
|
|
78
|
-
|
|
79
|
-
"act" => "deviceAction",
|
|
80
|
-
"app" => "applicationProtocol",
|
|
81
|
-
"c6a1" => "deviceCustomIPv6Address1",
|
|
82
|
-
"c6a1Label" => "deviceCustomIPv6Address1Label",
|
|
83
|
-
"c6a2" => "deviceCustomIPv6Address2",
|
|
84
|
-
"c6a2Label" => "deviceCustomIPv6Address2Label",
|
|
85
|
-
"c6a3" => "deviceCustomIPv6Address3",
|
|
86
|
-
"c6a3Label" => "deviceCustomIPv6Address3Label",
|
|
87
|
-
"c6a4" => "deviceCustomIPv6Address4",
|
|
88
|
-
"c6a4Label" => "deviceCustomIPv6Address4Label",
|
|
89
|
-
"cat" => "deviceEventCategory",
|
|
90
|
-
"cfp1" => "deviceCustomFloatingPoint1",
|
|
91
|
-
"cfp1Label" => "deviceCustomFloatingPoint1Label",
|
|
92
|
-
"cfp2" => "deviceCustomFloatingPoint2",
|
|
93
|
-
"cfp2Label" => "deviceCustomFloatingPoint2Label",
|
|
94
|
-
"cfp3" => "deviceCustomFloatingPoint3",
|
|
95
|
-
"cfp3Label" => "deviceCustomFloatingPoint3Label",
|
|
96
|
-
"cfp4" => "deviceCustomFloatingPoint4",
|
|
97
|
-
"cfp4Label" => "deviceCustomFloatingPoint4Label",
|
|
98
|
-
"cn1" => "deviceCustomNumber1",
|
|
99
|
-
"cn1Label" => "deviceCustomNumber1Label",
|
|
100
|
-
"cn2" => "deviceCustomNumber2",
|
|
101
|
-
"cn2Label" => "deviceCustomNumber2Label",
|
|
102
|
-
"cn3" => "deviceCustomNumber3",
|
|
103
|
-
"cn3Label" => "deviceCustomNumber3Label",
|
|
104
|
-
"cnt" => "baseEventCount",
|
|
105
|
-
"cs1" => "deviceCustomString1",
|
|
106
|
-
"cs1Label" => "deviceCustomString1Label",
|
|
107
|
-
"cs2" => "deviceCustomString2",
|
|
108
|
-
"cs2Label" => "deviceCustomString2Label",
|
|
109
|
-
"cs3" => "deviceCustomString3",
|
|
110
|
-
"cs3Label" => "deviceCustomString3Label",
|
|
111
|
-
"cs4" => "deviceCustomString4",
|
|
112
|
-
"cs4Label" => "deviceCustomString4Label",
|
|
113
|
-
"cs5" => "deviceCustomString5",
|
|
114
|
-
"cs5Label" => "deviceCustomString5Label",
|
|
115
|
-
"cs6" => "deviceCustomString6",
|
|
116
|
-
"cs6Label" => "deviceCustomString6Label",
|
|
117
|
-
"dhost" => "destinationHostName",
|
|
118
|
-
"dmac" => "destinationMacAddress",
|
|
119
|
-
"dntdom" => "destinationNtDomain",
|
|
120
|
-
"dpid" => "destinationProcessId",
|
|
121
|
-
"dpriv" => "destinationUserPrivileges",
|
|
122
|
-
"dproc" => "destinationProcessName",
|
|
123
|
-
"dpt" => "destinationPort",
|
|
124
|
-
"dst" => "destinationAddress",
|
|
125
|
-
"duid" => "destinationUserId",
|
|
126
|
-
"duser" => "destinationUserName",
|
|
127
|
-
"dvc" => "deviceAddress",
|
|
128
|
-
"dvchost" => "deviceHostName",
|
|
129
|
-
"dvcpid" => "deviceProcessId",
|
|
130
|
-
"end" => "endTime",
|
|
131
|
-
"fname" => "fileName",
|
|
132
|
-
"fsize" => "fileSize",
|
|
133
|
-
"in" => "bytesIn",
|
|
134
|
-
"msg" => "message",
|
|
135
|
-
"out" => "bytesOut",
|
|
136
|
-
"outcome" => "eventOutcome",
|
|
137
|
-
"proto" => "transportProtocol",
|
|
138
|
-
"request" => "requestUrl",
|
|
139
|
-
"rt" => "deviceReceiptTime",
|
|
140
|
-
"shost" => "sourceHostName",
|
|
141
|
-
"smac" => "sourceMacAddress",
|
|
142
|
-
"sntdom" => "sourceNtDomain",
|
|
143
|
-
"spid" => "sourceProcessId",
|
|
144
|
-
"spriv" => "sourceUserPrivileges",
|
|
145
|
-
"sproc" => "sourceProcessName",
|
|
146
|
-
"spt" => "sourcePort",
|
|
147
|
-
"src" => "sourceAddress",
|
|
148
|
-
"start" => "startTime",
|
|
149
|
-
"suid" => "sourceUserId",
|
|
150
|
-
"suser" => "sourceUserName",
|
|
151
|
-
"ahost" => "agentHostName",
|
|
152
|
-
"art" => "agentReceiptTime",
|
|
153
|
-
"at" => "agentType",
|
|
154
|
-
"aid" => "agentId",
|
|
155
|
-
"_cefVer" => "cefVersion",
|
|
156
|
-
"agt" => "agentAddress",
|
|
157
|
-
"av" => "agentVersion",
|
|
158
|
-
"atz" => "agentTimeZone",
|
|
159
|
-
"dtz" => "destinationTimeZone",
|
|
160
|
-
"slong" => "sourceLongitude",
|
|
161
|
-
"slat" => "sourceLatitude",
|
|
162
|
-
"dlong" => "destinationLongitude",
|
|
163
|
-
"dlat" => "destinationLatitude",
|
|
164
|
-
"catdt" => "categoryDeviceType",
|
|
165
|
-
"mrt" => "managerReceiptTime",
|
|
166
|
-
"amac" => "agentMacAddress"
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
# Reverse mapping of CEF full field names to CEF extensions field names for encoding into a CEF event for output.
|
|
170
|
-
REVERSE_MAPPINGS = MAPPINGS.invert
|
|
92
|
+
# Defines whether a set of device-specific CEF fields represent the _observer_,
|
|
93
|
+
# or the actual `host` on which the event occurred. If this codec handles a mix,
|
|
94
|
+
# it is safe to use the default `observer`.
|
|
95
|
+
config :device, :validate => %w(observer host), :default => 'observer'
|
|
171
96
|
|
|
172
97
|
# A CEF Header is a sequence of zero or more:
|
|
173
98
|
# - backslash-escaped pipes; OR
|
|
@@ -255,6 +180,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
255
180
|
@delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n")
|
|
256
181
|
@buffer = FileWatch::BufferedTokenizer.new(@delimiter)
|
|
257
182
|
end
|
|
183
|
+
|
|
184
|
+
require_relative 'cef/timestamp_normalizer'
|
|
185
|
+
@timestamp_normalzer = TimestampNormalizer.new(locale: @locale, timezone: @default_timezone)
|
|
186
|
+
|
|
187
|
+
generate_header_fields!
|
|
188
|
+
generate_mappings!
|
|
258
189
|
end
|
|
259
190
|
|
|
260
191
|
public
|
|
@@ -286,7 +217,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
286
217
|
|
|
287
218
|
# Use a scanning parser to capture the HEADER_FIELDS
|
|
288
219
|
unprocessed_data = data
|
|
289
|
-
|
|
220
|
+
@header_fields.each do |field_name|
|
|
290
221
|
match_data = HEADER_SCANNER.match(unprocessed_data)
|
|
291
222
|
break if match_data.nil? # missing fields
|
|
292
223
|
|
|
@@ -304,22 +235,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
304
235
|
message = unprocessed_data
|
|
305
236
|
|
|
306
237
|
# Try and parse out the syslog header if there is one
|
|
307
|
-
|
|
238
|
+
cef_version_field = @header_fields[0]
|
|
239
|
+
if (cef_version = event.get(cef_version_field)).include?(' ')
|
|
308
240
|
split_cef_version = cef_version.rpartition(' ')
|
|
309
|
-
event.set(
|
|
310
|
-
event.set(
|
|
241
|
+
event.set(@syslog_header, split_cef_version[0])
|
|
242
|
+
event.set(cef_version_field, split_cef_version[2])
|
|
311
243
|
end
|
|
312
244
|
|
|
313
245
|
# Get rid of the CEF bit in the version
|
|
314
|
-
event.set(
|
|
246
|
+
event.set(cef_version_field, delete_cef_prefix(event.get(cef_version_field)))
|
|
315
247
|
|
|
316
248
|
# Use a scanning parser to capture the Extension Key/Value Pairs
|
|
317
249
|
if message && message.include?('=')
|
|
318
250
|
message = message.strip
|
|
251
|
+
extension_fields = {}
|
|
319
252
|
|
|
320
253
|
message.scan(EXTENSION_KEY_VALUE_SCANNER) do |extension_field_key, raw_extension_field_value|
|
|
321
254
|
# expand abbreviated extension field keys
|
|
322
|
-
extension_field_key =
|
|
255
|
+
extension_field_key = @decode_mapping.fetch(extension_field_key, extension_field_key)
|
|
323
256
|
|
|
324
257
|
# convert extension field name to strict legal field_reference, fixing field names with ambiguous array-like syntax
|
|
325
258
|
extension_field_key = extension_field_key.sub(EXTENSION_KEY_ARRAY_CAPTURE, '[\1]\2') if extension_field_key.end_with?(']')
|
|
@@ -327,7 +260,21 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
327
260
|
# process legal extension field value escapes
|
|
328
261
|
extension_field_value = raw_extension_field_value.gsub(EXTENSION_VALUE_ESCAPE_CAPTURE, '\1')
|
|
329
262
|
|
|
330
|
-
|
|
263
|
+
extension_fields[extension_field_key] = extension_field_value
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# in ECS mode, normalize timestamps including timezone.
|
|
267
|
+
if ecs_compatibility != :disabled
|
|
268
|
+
device_timezone = extension_fields['[event][timezone]']
|
|
269
|
+
@timestamp_fields.each do |timestamp_field_name|
|
|
270
|
+
raw_timestamp = extension_fields.delete(timestamp_field_name) or next
|
|
271
|
+
value = normalize_timestamp(raw_timestamp, device_timezone)
|
|
272
|
+
event.set(timestamp_field_name, value)
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
extension_fields.each do |field_key, field_value|
|
|
277
|
+
event.set(field_key, field_value)
|
|
331
278
|
end
|
|
332
279
|
end
|
|
333
280
|
|
|
@@ -368,6 +315,234 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
368
315
|
|
|
369
316
|
private
|
|
370
317
|
|
|
318
|
+
def generate_header_fields!
|
|
319
|
+
# @header_fields is an _ordered_ set of fields.
|
|
320
|
+
@header_fields = [
|
|
321
|
+
ecs_select[disabled: 'cefVersion', v1: '[cef][version]'],
|
|
322
|
+
ecs_select[disabled: 'deviceVendor', v1: '[observer][vendor]'],
|
|
323
|
+
ecs_select[disabled: 'deviceProduct', v1: '[observer][product]'],
|
|
324
|
+
ecs_select[disabled: 'deviceVersion', v1: '[observer][version]'],
|
|
325
|
+
ecs_select[disabled: 'deviceEventClassId', v1: '[event][code]'],
|
|
326
|
+
ecs_select[disabled: 'name', v1: '[cef][name]'],
|
|
327
|
+
ecs_select[disabled: 'severity', v1: '[event][severity]']
|
|
328
|
+
].map(&:freeze).freeze
|
|
329
|
+
# the @syslog_header is the field name used when a syslog header preceeds the CEF Version.
|
|
330
|
+
@syslog_header = ecs_select[disabled:'syslog',v1:'[log][syslog][header]']
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
class CEFField
|
|
334
|
+
##
|
|
335
|
+
# @param name [String]: the full CEF name of a field
|
|
336
|
+
# @param key [String] (optional): an abbreviated CEF key to use when encoding a value with `reverse_mapping => true`
|
|
337
|
+
# when left unspecified, the `key` is the field's `name`.
|
|
338
|
+
# @param ecs_field [String] (optional): an ECS-compatible field reference to use, with square-bracket syntax.
|
|
339
|
+
# when left unspecified, the `ecs_field` is the field's `name`.
|
|
340
|
+
# @param legacy [String] (optional): a legacy CEF name to support in pass-through.
|
|
341
|
+
# in decoding mode without ECS, field name will be used as-provided.
|
|
342
|
+
# in encoding mode without ECS when provided to `fields` and `reverse_mapping => false`,
|
|
343
|
+
# field name will be used as-provided.
|
|
344
|
+
# @param priority [Integer] (optional): when multiple fields resolve to the same ECS field name, the field with the
|
|
345
|
+
# highest `prioriry` will be used by the encoder.
|
|
346
|
+
def initialize(name, key: name, ecs_field: name, legacy:nil, priority:0, normalize:nil)
|
|
347
|
+
@name = name
|
|
348
|
+
@key = key
|
|
349
|
+
@ecs_field = ecs_field
|
|
350
|
+
@legacy = legacy
|
|
351
|
+
@priority = priority
|
|
352
|
+
@normalize = normalize
|
|
353
|
+
end
|
|
354
|
+
attr_reader :name
|
|
355
|
+
attr_reader :key
|
|
356
|
+
attr_reader :ecs_field
|
|
357
|
+
attr_reader :legacy
|
|
358
|
+
attr_reader :priority
|
|
359
|
+
attr_reader :normalize
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
def generate_mappings!
|
|
363
|
+
encode_mapping = Hash.new
|
|
364
|
+
decode_mapping = Hash.new
|
|
365
|
+
timestamp_fields = Set.new
|
|
366
|
+
[
|
|
367
|
+
CEFField.new("agentAddress", key: "agt", ecs_field: "[agent][ip]"),
|
|
368
|
+
CEFField.new("agentDnsDomain", ecs_field: "[cef][agent][registered_domain]", priority: 10),
|
|
369
|
+
CEFField.new("agentHostName", key: "ahost", ecs_field: "[agent][name]"),
|
|
370
|
+
CEFField.new("agentId", key: "aid", ecs_field: "[agent][id]"),
|
|
371
|
+
CEFField.new("agentMacAddress", key: "amac", ecs_field: "[agent][mac]"),
|
|
372
|
+
CEFField.new("agentNtDomain", ecs_field: "[cef][agent][registered_domain]"),
|
|
373
|
+
CEFField.new("agentReceiptTime", key: "art", ecs_field: "[event][created]", normalize: :timestamp),
|
|
374
|
+
CEFField.new("agentTimeZone", key: "atz", ecs_field: "[cef][agent][timezone]"),
|
|
375
|
+
CEFField.new("agentTranslatedAddress", ecs_field: "[cef][agent][nat][ip]"),
|
|
376
|
+
CEFField.new("agentTranslatedZoneExternalID", ecs_field: "[cef][agent][translated_zone][external_id]"),
|
|
377
|
+
CEFField.new("agentTranslatedZoneURI", ecs_field: "[cef][agent][translated_zone][uri]"),
|
|
378
|
+
CEFField.new("agentType", key: "at", ecs_field: "[agent][type]"),
|
|
379
|
+
CEFField.new("agentVersion", key: "av", ecs_field: "[agent][version]"),
|
|
380
|
+
CEFField.new("agentZoneExternalID", ecs_field: "[cef][agent][zone][external_id]"),
|
|
381
|
+
CEFField.new("agentZoneURI", ecs_field: "[cef][agent][zone][uri]"),
|
|
382
|
+
CEFField.new("applicationProtocol", key: "app", ecs_field: "[network][protocol]"),
|
|
383
|
+
CEFField.new("baseEventCount", key: "cnt", ecs_field: "[cef][base_event_count]"),
|
|
384
|
+
CEFField.new("bytesIn", key: "in", ecs_field: "[source][bytes]"),
|
|
385
|
+
CEFField.new("bytesOut", key: "out", ecs_field: "[destination][bytes]"),
|
|
386
|
+
CEFField.new("categoryDeviceType", key: "catdt", ecs_field: "[cef][device_type]"),
|
|
387
|
+
CEFField.new("customerExternalID", ecs_field: "[organization][id]"),
|
|
388
|
+
CEFField.new("customerURI", ecs_field: "[organization][name]"),
|
|
389
|
+
CEFField.new("destinationAddress", key: "dst", ecs_field: "[destination][ip]"),
|
|
390
|
+
CEFField.new("destinationDnsDomain", ecs_field: "[destination][registered_domain]", priority: 10),
|
|
391
|
+
CEFField.new("destinationGeoLatitude", key: "dlat", ecs_field: "[destination][geo][location][lat]", legacy: "destinationLatitude"),
|
|
392
|
+
CEFField.new("destinationGeoLongitude", key: "dlong", ecs_field: "[destination][geo][location][lon]", legacy: "destinationLongitude"),
|
|
393
|
+
CEFField.new("destinationHostName", key: "dhost", ecs_field: "[destination][domain]"),
|
|
394
|
+
CEFField.new("destinationMacAddress", key: "dmac", ecs_field: "[destination][mac]"),
|
|
395
|
+
CEFField.new("destinationNtDomain", key: "dntdom", ecs_field: "[destination][registered_domain]"),
|
|
396
|
+
CEFField.new("destinationPort", key: "dpt", ecs_field: "[destination][port]"),
|
|
397
|
+
CEFField.new("destinationProcessId", key: "dpid", ecs_field: "[destination][process][pid]"),
|
|
398
|
+
CEFField.new("destinationProcessName", key: "dproc", ecs_field: "[destination][process][name]"),
|
|
399
|
+
CEFField.new("destinationServiceName", ecs_field: "[destination][service][name]"),
|
|
400
|
+
CEFField.new("destinationTranslatedAddress", ecs_field: "[destination][nat][ip]"),
|
|
401
|
+
CEFField.new("destinationTranslatedPort", ecs_field: "[destination][nat][port]"),
|
|
402
|
+
CEFField.new("destinationTranslatedZoneExternalID", ecs_field: "[cef][destination][translated_zone][external_id]"),
|
|
403
|
+
CEFField.new("destinationTranslatedZoneURI", ecs_field: "[cef][destination][translated_zone][uri]"),
|
|
404
|
+
CEFField.new("destinationUserId", key: "duid", ecs_field: "[destination][user][id]"),
|
|
405
|
+
CEFField.new("destinationUserName", key: "duser", ecs_field: "[destination][user][name]"),
|
|
406
|
+
CEFField.new("destinationUserPrivileges", key: "dpriv", ecs_field: "[destination][user][group][name]"),
|
|
407
|
+
CEFField.new("destinationZoneExternalID", ecs_field: "[cef][destination][zone][external_id]"),
|
|
408
|
+
CEFField.new("destinationZoneURI", ecs_field: "[cef][destination][zone][uri]"),
|
|
409
|
+
CEFField.new("deviceAction", key: "act", ecs_field: "[event][action]"),
|
|
410
|
+
CEFField.new("deviceAddress", key: "dvc", ecs_field: "[#{@device}][ip]"),
|
|
411
|
+
CEFField.new("deviceCustomFloatingPoint1", key: "cfp1", ecs_field: "[cef][device_custom_floating_point_1][value]"),
|
|
412
|
+
CEFField.new("deviceCustomFloatingPoint1Label", key: "cfp1Label", ecs_field: "[cef][device_custom_floating_point_1][label]"),
|
|
413
|
+
CEFField.new("deviceCustomFloatingPoint2", key: "cfp2", ecs_field: "[cef][device_custom_floating_point_2][value]"),
|
|
414
|
+
CEFField.new("deviceCustomFloatingPoint2Label", key: "cfp2Label", ecs_field: "[cef][device_custom_floating_point_2][label]"),
|
|
415
|
+
CEFField.new("deviceCustomFloatingPoint3", key: "cfp3", ecs_field: "[cef][device_custom_floating_point_3][value]"),
|
|
416
|
+
CEFField.new("deviceCustomFloatingPoint3Label", key: "cfp3Label", ecs_field: "[cef][device_custom_floating_point_3][label]"),
|
|
417
|
+
CEFField.new("deviceCustomFloatingPoint4", key: "cfp4", ecs_field: "[cef][device_custom_floating_point_4][value]"),
|
|
418
|
+
CEFField.new("deviceCustomFloatingPoint4Label", key: "cfp4Label", ecs_field: "[cef][device_custom_floating_point_4][label]"),
|
|
419
|
+
CEFField.new("deviceCustomIPv6Address1", key: "c6a1", ecs_field: "[cef][device_custom_ipv6_address_1][value]"),
|
|
420
|
+
CEFField.new("deviceCustomIPv6Address1Label", key: "c6a1Label", ecs_field: "[cef][device_custom_ipv6_address_1][label]"),
|
|
421
|
+
CEFField.new("deviceCustomIPv6Address2", key: "c6a2", ecs_field: "[cef][device_custom_ipv6_address_2][value]"),
|
|
422
|
+
CEFField.new("deviceCustomIPv6Address2Label", key: "c6a2Label", ecs_field: "[cef][device_custom_ipv6_address_2][label]"),
|
|
423
|
+
CEFField.new("deviceCustomIPv6Address3", key: "c6a3", ecs_field: "[cef][device_custom_ipv6_address_3][value]"),
|
|
424
|
+
CEFField.new("deviceCustomIPv6Address3Label", key: "c6a3Label", ecs_field: "[cef][device_custom_ipv6_address_3][label]"),
|
|
425
|
+
CEFField.new("deviceCustomIPv6Address4", key: "c6a4", ecs_field: "[cef][device_custom_ipv6_address_4][value]"),
|
|
426
|
+
CEFField.new("deviceCustomIPv6Address4Label", key: "c6a4Label", ecs_field: "[cef][device_custom_ipv6_address_4][label]"),
|
|
427
|
+
CEFField.new("deviceCustomNumber1", key: "cn1", ecs_field: "[cef][device_custom_number_1][value]"),
|
|
428
|
+
CEFField.new("deviceCustomNumber1Label", key: "cn1Label", ecs_field: "[cef][device_custom_number_1][label]"),
|
|
429
|
+
CEFField.new("deviceCustomNumber2", key: "cn2", ecs_field: "[cef][device_custom_number_2][value]"),
|
|
430
|
+
CEFField.new("deviceCustomNumber2Label", key: "cn2Label", ecs_field: "[cef][device_custom_number_2][label]"),
|
|
431
|
+
CEFField.new("deviceCustomNumber3", key: "cn3", ecs_field: "[cef][device_custom_number_3][value]"),
|
|
432
|
+
CEFField.new("deviceCustomNumber3Label", key: "cn3Label", ecs_field: "[cef][device_custom_number_3][label]"),
|
|
433
|
+
CEFField.new("deviceCustomString1", key: "cs1", ecs_field: "[cef][device_custom_string_1][value]"),
|
|
434
|
+
CEFField.new("deviceCustomString1Label", key: "cs1Label", ecs_field: "[cef][device_custom_string_1][label]"),
|
|
435
|
+
CEFField.new("deviceCustomString2", key: "cs2", ecs_field: "[cef][device_custom_string_2][value]"),
|
|
436
|
+
CEFField.new("deviceCustomString2Label", key: "cs2Label", ecs_field: "[cef][device_custom_string_2][label]"),
|
|
437
|
+
CEFField.new("deviceCustomString3", key: "cs3", ecs_field: "[cef][device_custom_string_3][value]"),
|
|
438
|
+
CEFField.new("deviceCustomString3Label", key: "cs3Label", ecs_field: "[cef][device_custom_string_3][label]"),
|
|
439
|
+
CEFField.new("deviceCustomString4", key: "cs4", ecs_field: "[cef][device_custom_string_4][value]"),
|
|
440
|
+
CEFField.new("deviceCustomString4Label", key: "cs4Label", ecs_field: "[cef][device_custom_string_4][label]"),
|
|
441
|
+
CEFField.new("deviceCustomString5", key: "cs5", ecs_field: "[cef][device_custom_string_5][value]"),
|
|
442
|
+
CEFField.new("deviceCustomString5Label", key: "cs5Label", ecs_field: "[cef][device_custom_string_5][label]"),
|
|
443
|
+
CEFField.new("deviceCustomString6", key: "cs6", ecs_field: "[cef][device_custom_string_6][value]"),
|
|
444
|
+
CEFField.new("deviceCustomString6Label", key: "cs6Label", ecs_field: "[cef][device_custom_string_6][label]"),
|
|
445
|
+
CEFField.new("deviceDirection", ecs_field: "[network][direction]"),
|
|
446
|
+
CEFField.new("deviceDnsDomain", ecs_field: "[#{@device}][registered_domain]", priority: 10),
|
|
447
|
+
CEFField.new("deviceEventCategory", key: "cat", ecs_field: "[cef][category]"),
|
|
448
|
+
CEFField.new("deviceExternalId", ecs_field: (@device == 'host' ? "[host][id]" : "[observer][name]")),
|
|
449
|
+
CEFField.new("deviceFacility", ecs_field: "[log][syslog][facility][code]"),
|
|
450
|
+
CEFField.new("deviceHostName", key: "dvchost", ecs_field: (@device == 'host' ? '[host][name]' : '[observer][hostname]')),
|
|
451
|
+
CEFField.new("deviceInboundInterface", ecs_field: "[observer][ingress][interface][name]"),
|
|
452
|
+
CEFField.new("deviceMacAddress", key: "dvcmac", ecs_field: "[@device][mac]"),
|
|
453
|
+
CEFField.new("deviceNtDomain", ecs_field: "[cef][nt_domain]"),
|
|
454
|
+
CEFField.new("deviceOutboundInterface", ecs_field: "[observer][egress][interface][name]"),
|
|
455
|
+
CEFField.new("devicePayloadId", ecs_field: "[cef][payload_id]"),
|
|
456
|
+
CEFField.new("deviceProcessId", key: "dvcpid", ecs_field: "[process][pid]"),
|
|
457
|
+
CEFField.new("deviceProcessName", ecs_field: "[process][name]"),
|
|
458
|
+
CEFField.new("deviceReceiptTime", key: "rt", ecs_field: "@timestamp", normalize: :timestamp),
|
|
459
|
+
CEFField.new("deviceTimeZone", key: "dtz", ecs_field: "[event][timezone]", legacy: "destinationTimeZone"),
|
|
460
|
+
CEFField.new("deviceTranslatedAddress", ecs_field: "[host][nat][ip]"),
|
|
461
|
+
CEFField.new("deviceTranslatedZoneExternalID", ecs_field: "[cef][translated_zone][external_id]"),
|
|
462
|
+
CEFField.new("deviceTranslatedZoneURI", ecs_field: "[cef][translated_zone][uri]"),
|
|
463
|
+
CEFField.new("deviceVersion", ecs_field: "[observer][version]"),
|
|
464
|
+
CEFField.new("deviceZoneExternalID", ecs_field: "[cef][zone][external_id]"),
|
|
465
|
+
CEFField.new("deviceZoneURI", ecs_field: "[cef][zone][uri]"),
|
|
466
|
+
CEFField.new("endTime", key: "end", ecs_field: "[event][end]", normalize: :timestamp),
|
|
467
|
+
CEFField.new("eventId", ecs_field: "[event][id]"),
|
|
468
|
+
CEFField.new("eventOutcome", key: "outcome", ecs_field: "[event][outcome]"),
|
|
469
|
+
CEFField.new("externalId", ecs_field: "[cef][external_id]"),
|
|
470
|
+
CEFField.new("fileCreateTime", ecs_field: "[file][created]"),
|
|
471
|
+
CEFField.new("fileHash", ecs_field: "[file][hash]]"),
|
|
472
|
+
CEFField.new("fileId", ecs_field: "[file][inode]"),
|
|
473
|
+
CEFField.new("fileModificationTime", ecs_field: "[file][mtime]", normalize: :timestamp),
|
|
474
|
+
CEFField.new("fileName", key: "fname", ecs_field: "[file][name]"),
|
|
475
|
+
CEFField.new("filePath", ecs_field: "[file][path]"),
|
|
476
|
+
CEFField.new("filePermission", ecs_field: "[file][group]"),
|
|
477
|
+
CEFField.new("fileSize", key: "fsize", ecs_field: "[file][size]"),
|
|
478
|
+
CEFField.new("fileType", ecs_field: "[file][extension]"),
|
|
479
|
+
CEFField.new("managerReceiptTime", key: "mrt", ecs_field: "[event][ingested]", normalize: :timestamp),
|
|
480
|
+
CEFField.new("message", key: "msg", ecs_field: "[message]"),
|
|
481
|
+
CEFField.new("oldFileCreateTime", ecs_field: "[cef][old_file][created]", normalize: :timestamp),
|
|
482
|
+
CEFField.new("oldFileHash", ecs_field: "[cef][old_file][hash]"),
|
|
483
|
+
CEFField.new("oldFileId", ecs_field: "[cef][old_file][inode]"),
|
|
484
|
+
CEFField.new("oldFileModificationTime", ecs_field: "[cef][old_file][mtime]", normalize: :timestamp),
|
|
485
|
+
CEFField.new("oldFileName", ecs_field: "[cef][old_file][name]"),
|
|
486
|
+
CEFField.new("oldFilePath", ecs_field: "[cef][old_file][path]"),
|
|
487
|
+
CEFField.new("oldFilePermission", ecs_field: "[cef][old_file][group]"),
|
|
488
|
+
CEFField.new("oldFileSize", ecs_field: "[cef][old_file][size]"),
|
|
489
|
+
CEFField.new("oldFileType", ecs_field: "[cef][old_file][extension]"),
|
|
490
|
+
CEFField.new("rawEvent", ecs_field: "[event][original]"),
|
|
491
|
+
CEFField.new("Reason", key: "reason", ecs_field: "[event][reason]"),
|
|
492
|
+
CEFField.new("requestClientApplication", ecs_field: "[user_agent][original]"),
|
|
493
|
+
CEFField.new("requestContext", ecs_field: "[http][request][referrer]"),
|
|
494
|
+
CEFField.new("requestCookies", ecs_field: "[cef][request][cookies]"),
|
|
495
|
+
CEFField.new("requestMethod", ecs_field: "[http][request][method]"),
|
|
496
|
+
CEFField.new("requestUrl", key: "request", ecs_field: "[url][original]"),
|
|
497
|
+
CEFField.new("sourceAddress", key: "src", ecs_field: "[source][ip]"),
|
|
498
|
+
CEFField.new("sourceDnsDomain", ecs_field: "[source][registered_domain]", priority: 10),
|
|
499
|
+
CEFField.new("sourceGeoLatitude", key: "slat", ecs_field: "[source][geo][location][lat]", legacy: "sourceLatitude"),
|
|
500
|
+
CEFField.new("sourceGeoLongitude", key: "slong", ecs_field: "[source][geo][location][lon]", legacy: "sourceLongitude"),
|
|
501
|
+
CEFField.new("sourceHostName", key: "shost", ecs_field: "[source][domain]"),
|
|
502
|
+
CEFField.new("sourceMacAddress", key: "smac", ecs_field: "[source][mac]"),
|
|
503
|
+
CEFField.new("sourceNtDomain", key: "sntdom", ecs_field: "[source][registered_domain]"),
|
|
504
|
+
CEFField.new("sourcePort", key: "spt", ecs_field: "[source][port]"),
|
|
505
|
+
CEFField.new("sourceProcessId", key: "spid", ecs_field: "[source][process][pid]"),
|
|
506
|
+
CEFField.new("sourceProcessName", key: "sproc", ecs_field: "[source][process][name]"),
|
|
507
|
+
CEFField.new("sourceServiceName", ecs_field: "[source][service][name]"),
|
|
508
|
+
CEFField.new("sourceTranslatedAddress", ecs_field: "[source][nat][ip]"),
|
|
509
|
+
CEFField.new("sourceTranslatedPort", ecs_field: "[source][nat][port]"),
|
|
510
|
+
CEFField.new("sourceTranslatedZoneExternalID", ecs_field: "[cef][source][translated_zone][external_id]"),
|
|
511
|
+
CEFField.new("sourceTranslatedZoneURI", ecs_field: "[cef][source][translated_zone][uri]"),
|
|
512
|
+
CEFField.new("sourceUserId", key: "suid", ecs_field: "[source][user][id]"),
|
|
513
|
+
CEFField.new("sourceUserName", key: "suser", ecs_field: "[source][user][name]"),
|
|
514
|
+
CEFField.new("sourceUserPrivileges", key: "spriv", ecs_field: "[source][user][group][name]"),
|
|
515
|
+
CEFField.new("sourceZoneExternalID", ecs_field: "[cef][source][zone][external_id]"),
|
|
516
|
+
CEFField.new("sourceZoneURI", ecs_field: "[cef][source][zone][uri]"),
|
|
517
|
+
CEFField.new("startTime", key: "start", ecs_field: "[event][start]", normalize: :timestamp),
|
|
518
|
+
CEFField.new("transportProtocol", key: "proto", ecs_field: "[network][transport]"),
|
|
519
|
+
CEFField.new("type", ecs_field: "[cef][type]"),
|
|
520
|
+
].sort_by(&:priority).each do |cef|
|
|
521
|
+
field_name = ecs_select[disabled:cef.name, v1:cef.ecs_field]
|
|
522
|
+
|
|
523
|
+
# whether the source is a cef_key or cef_name, normalize to field_name
|
|
524
|
+
decode_mapping[cef.key] = field_name
|
|
525
|
+
decode_mapping[cef.name] = field_name
|
|
526
|
+
|
|
527
|
+
# whether source is a cef_name or a field_name, normalize to target
|
|
528
|
+
normalized_encode_target = @reverse_mapping ? cef.key : cef.name
|
|
529
|
+
encode_mapping[field_name] = normalized_encode_target
|
|
530
|
+
encode_mapping[cef.name] = normalized_encode_target unless cef.name == field_name
|
|
531
|
+
|
|
532
|
+
# if a field has an alias, normalize pass-through
|
|
533
|
+
if cef.legacy
|
|
534
|
+
decode_mapping[cef.legacy] = ecs_select[disabled:cef.legacy, v1:cef.ecs_field]
|
|
535
|
+
encode_mapping[cef.legacy] = @reverse_mapping ? cef.key : cef.legacy
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
timestamp_fields << field_name if ecs_compatibility != :disabled && cef.normalize == :timestamp
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
@decode_mapping = decode_mapping.dup.freeze
|
|
542
|
+
@encode_mapping = encode_mapping.dup.freeze
|
|
543
|
+
@timestamp_fields = timestamp_fields.dup.freeze
|
|
544
|
+
end
|
|
545
|
+
|
|
371
546
|
# Escape pipes and backslashes in the header. Equal signs are ok.
|
|
372
547
|
# Newlines are forbidden.
|
|
373
548
|
def sanitize_header_field(value)
|
|
@@ -392,17 +567,23 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
392
567
|
.gsub(EXTENSION_VALUE_SANITIZER_PATTERN, EXTENSION_VALUE_SANITIZER_MAPPING)
|
|
393
568
|
end
|
|
394
569
|
|
|
570
|
+
def normalize_timestamp(value, device_timezone_name)
|
|
571
|
+
value = @timestamp_normalzer.normalize(value, device_timezone_name).iso8601(9)
|
|
572
|
+
|
|
573
|
+
LogStash::Timestamp.new(value)
|
|
574
|
+
rescue => e
|
|
575
|
+
@logger.error("Failed to parse CEF timestamp value `#{value}` (#{e.message})")
|
|
576
|
+
raise InvalidTimestamp.new("Not a valid CEF timestamp: `#{value}`")
|
|
577
|
+
end
|
|
578
|
+
|
|
395
579
|
def get_value(fieldname, event)
|
|
396
580
|
val = event.get(fieldname)
|
|
397
581
|
|
|
398
582
|
return nil if val.nil?
|
|
399
583
|
|
|
400
|
-
key =
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
key = REVERSE_MAPPINGS[key] || key
|
|
404
|
-
end
|
|
405
|
-
|
|
584
|
+
key = @encode_mapping.fetch(fieldname, fieldname)
|
|
585
|
+
key = sanitize_extension_key(key)
|
|
586
|
+
|
|
406
587
|
case val
|
|
407
588
|
when Array, Hash
|
|
408
589
|
return "#{key}=#{sanitize_extension_val(val.to_json)}"
|