logstash-codec-cef 6.1.2-java → 6.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +122 -8
- data/lib/logstash/codecs/cef.rb +290 -109
- data/lib/logstash/codecs/cef/timestamp_normalizer.rb +115 -0
- data/logstash-codec-cef.gemspec +2 -1
- data/spec/codecs/cef/timestamp_normalizer_spec.rb +274 -0
- data/spec/codecs/cef_spec.rb +477 -306
- metadata +19 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 25264d450cdfa027ac9758a05972c0c87119a635238b9bacfc746e4daec40dff
|
|
4
|
+
data.tar.gz: 63dbd8558c231e61c9e55962484042a52c28177cba5b4adcf4d49291aace491a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 62ac45d798abaf3008f99357b578506237bddc024904951cab64bdb36fd61f1c91894e4e70f2ce6a911fde46c4d1a15c69a1368b2a0dd69a6f509e09e8ee8192
|
|
7
|
+
data.tar.gz: 46dd75f39b72ae788dfcd29f21810be7086bb16cb1984f81f50a786682d66f06f7e6a2ee543f3e88a3e8951da5680f99fdcd45d28e606c16171d34bc30ceb4a4
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
## 6.2.0
|
|
2
|
+
- Introduce ECS Compatibility mode [#83](https://github.com/logstash-plugins/logstash-codec-cef/pull/83).
|
|
3
|
+
|
|
1
4
|
## 6.1.2
|
|
2
5
|
- Added error log with full payload when something bad happens in decoding a message[#84](https://github.com/logstash-plugins/logstash-codec-cef/pull/84)
|
|
3
6
|
|
data/docs/index.asciidoc
CHANGED
|
@@ -27,14 +27,53 @@ https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-docum
|
|
|
27
27
|
If this codec receives a payload from an input that is not a valid CEF message, then it will
|
|
28
28
|
produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
|
29
29
|
|
|
30
|
+
==== Compatibility with the Elastic Common Schema (ECS)
|
|
31
|
+
|
|
32
|
+
This plugin can be used to decode CEF events _into_ the Elastic Common Schema, or to encode ECS-compatible events into CEF.
|
|
33
|
+
It can also be used _without_ ECS, encoding and decoding events using only CEF-defined field names and keys.
|
|
34
|
+
|
|
35
|
+
The ECS Compatibility mode for a specific plugin instance can be controlled by setting <<plugins-{type}s-{plugin}-ecs_compatibility>> when defining the codec:
|
|
36
|
+
|
|
37
|
+
[source,sh]
|
|
38
|
+
-----
|
|
39
|
+
input {
|
|
40
|
+
tcp {
|
|
41
|
+
# ...
|
|
42
|
+
codec => cef {
|
|
43
|
+
ecs_compatibility => v1
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
-----
|
|
48
|
+
|
|
49
|
+
If left unspecified, the value of the `pipeline.ecs_compatibility` setting is used.
|
|
50
|
+
|
|
51
|
+
===== Timestamps and ECS Compatiblity
|
|
52
|
+
|
|
53
|
+
When running in ECS Compatibility Mode, timestamp-type fields are parsed and normalized
|
|
54
|
+
to specific points on the timeline.
|
|
55
|
+
|
|
56
|
+
Because the CEF format allows ambiguous timestamp formats, some reasonable assumptions are made:
|
|
57
|
+
|
|
58
|
+
- When the timestamp does not include a year, we assume it happened in the recent past
|
|
59
|
+
(or _very_ near future to accommodate out-of-sync clocks and timezone offsets).
|
|
60
|
+
- When the timestamp does not include UTC-offset information, we use the event's
|
|
61
|
+
timezone (`dtz` or `deviceTimeZone` field), or fall through to this plugin's
|
|
62
|
+
<<plugins-{type}s-{plugin}-default_timezone>>.
|
|
63
|
+
- Localized timestamps are parsed using the provided <<plugins-{type}s-{plugin}-locale>>.
|
|
64
|
+
|
|
30
65
|
[id="plugins-{type}s-{plugin}-options"]
|
|
31
66
|
==== Cef Codec Configuration Options
|
|
32
67
|
|
|
33
68
|
[cols="<,<,<",options="header",]
|
|
34
69
|
|=======================================================================
|
|
35
70
|
|Setting |Input type|Required
|
|
71
|
+
| <<plugins-{type}s-{plugin}-default_timezone>> |<<string,string>>|No
|
|
36
72
|
| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
|
|
73
|
+
| <<plugins-{type}s-{plugin}-device>> |<<string,string>>|No
|
|
74
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
|
|
37
75
|
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
|
76
|
+
| <<plugins-{type}s-{plugin}-locale>> |<<string,string>>|No
|
|
38
77
|
| <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
|
|
39
78
|
| <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
|
|
40
79
|
| <<plugins-{type}s-{plugin}-reverse_mapping>> |<<boolean,boolean>>|No
|
|
@@ -46,6 +85,21 @@ produce an event with the payload as the 'message' field and a '_cefparsefailure
|
|
|
46
85
|
|
|
47
86
|
|
|
48
87
|
|
|
88
|
+
[id="plugins-{type}s-{plugin}-default_timezone"]
|
|
89
|
+
===== `default_timezone`
|
|
90
|
+
|
|
91
|
+
* Value type is <<string,string>>
|
|
92
|
+
* Supported values are:
|
|
93
|
+
** https://en.wikipedia.org/wiki/List_of_tz_database_time_zones[Timezone names] (such as `Europe/Moscow`, `America/Argentina/Buenos_Aires`)
|
|
94
|
+
** UTC Offsets (such as `-08:00`, `+03:00`)
|
|
95
|
+
* The default value is your system time zone
|
|
96
|
+
* This option has no effect when _encoding_.
|
|
97
|
+
|
|
98
|
+
When parsing timestamp fields in ECS mode and encountering timestamps that
|
|
99
|
+
do not contain UTC-offset information, the `deviceTimeZone` (`dtz`) field
|
|
100
|
+
from the CEF payload is used to interpret the given time. If the event does
|
|
101
|
+
not include timezone information, this `default_timezone` is used instead.
|
|
102
|
+
|
|
49
103
|
[id="plugins-{type}s-{plugin}-delimiter"]
|
|
50
104
|
===== `delimiter`
|
|
51
105
|
|
|
@@ -69,21 +123,71 @@ This setting allows the following character sequences to have special meaning:
|
|
|
69
123
|
* `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
|
|
70
124
|
* `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
|
71
125
|
|
|
126
|
+
[id="plugins-{type}s-{plugin}-device"]
|
|
127
|
+
===== `device`
|
|
128
|
+
|
|
129
|
+
* Value type is <<string,string>>
|
|
130
|
+
* Supported values are:
|
|
131
|
+
** `observer`: indicates that device-specific fields represent the device used to _observe_ the event.
|
|
132
|
+
** `host`: indicates that device-specific fields represent the device on which the event _occurred_.
|
|
133
|
+
* The default value for this setting is `observer`.
|
|
134
|
+
* Option has no effect when <<plugins-{type}s-{plugin}-ecs_compatibility,`ecs_compatibility => disabled`>>.
|
|
135
|
+
* Option has no effect when _encoding_
|
|
136
|
+
|
|
137
|
+
Defines a set of device-specific CEF fields as either representing the device on which an
|
|
138
|
+
event _occurred_, or merely the device from which the event was _observed_.
|
|
139
|
+
This causes the relevant fields to be routed to either the `host` or the `observer`
|
|
140
|
+
top-level groupings.
|
|
141
|
+
|
|
142
|
+
If the codec handles data from a variety of sources, the ECS recommendation is to use `observer`.
|
|
143
|
+
|
|
144
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
|
145
|
+
===== `ecs_compatibility`
|
|
146
|
+
|
|
147
|
+
* Value type is <<string,string>>
|
|
148
|
+
* Supported values are:
|
|
149
|
+
** `disabled`: uses CEF-defined field names in the event (e.g., `bytesIn`, `sourceAddress`)
|
|
150
|
+
** `v1`: supports ECS-compatible event fields (e.g., `[source][bytes]`, `[source][ip]`)
|
|
151
|
+
* Default value depends on which version of Logstash is running:
|
|
152
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
|
153
|
+
** Otherwise, the default value is `disabled`.
|
|
154
|
+
|
|
155
|
+
Controls this plugin's compatibility with the
|
|
156
|
+
{ecs-ref}[Elastic Common Schema (ECS)]
|
|
157
|
+
(ECS)].
|
|
158
|
+
|
|
72
159
|
[id="plugins-{type}s-{plugin}-fields"]
|
|
73
160
|
===== `fields`
|
|
74
161
|
|
|
75
162
|
* Value type is <<array,array>>
|
|
76
163
|
* Default value is `[]`
|
|
164
|
+
* Option has no effect when _decoding_
|
|
165
|
+
|
|
166
|
+
When this codec is used in an Output Plugin, a list of fields can be provided to be included in CEF extensions part as key/value pairs.
|
|
77
167
|
|
|
78
|
-
|
|
168
|
+
[id="plugins-{type}s-{plugin}-locale"]
|
|
169
|
+
===== `locale`
|
|
170
|
+
|
|
171
|
+
* Value type is <<string,string>>
|
|
172
|
+
* Supported values are:
|
|
173
|
+
** Abbreviated language_COUNTRY format (e.g., `en_GB`, `pt_BR`)
|
|
174
|
+
** Valid https://tools.ietf.org/html/bcp47[IETF BCP 47] language tag (e.g., `zh-cmn-Hans-CN`)
|
|
175
|
+
* The default value is your system locale
|
|
176
|
+
* Option has no effect when _encoding_
|
|
177
|
+
|
|
178
|
+
When parsing timestamp fields in ECS mode and encountering timestamps in
|
|
179
|
+
a localized format, this `locale` is used to interpret locale-specific strings
|
|
180
|
+
such as month abbreviations.
|
|
79
181
|
|
|
80
182
|
[id="plugins-{type}s-{plugin}-name"]
|
|
81
183
|
===== `name`
|
|
82
184
|
|
|
83
185
|
* Value type is <<string,string>>
|
|
84
186
|
* Default value is `"Logstash"`
|
|
187
|
+
* Option has no effect when _decoding_
|
|
85
188
|
|
|
86
|
-
|
|
189
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
190
|
+
value of the name field in the CEF header. The new value can include `%{foo}` strings
|
|
87
191
|
to help you build a new value from other parts of the event.
|
|
88
192
|
|
|
89
193
|
[id="plugins-{type}s-{plugin}-product"]
|
|
@@ -91,8 +195,10 @@ to help you build a new value from other parts of the event.
|
|
|
91
195
|
|
|
92
196
|
* Value type is <<string,string>>
|
|
93
197
|
* Default value is `"Logstash"`
|
|
198
|
+
* Option has no effect when _decoding_
|
|
94
199
|
|
|
95
|
-
|
|
200
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
201
|
+
value of the device product field in CEF header. The new value can include `%{foo}` strings
|
|
96
202
|
to help you build a new value from other parts of the event.
|
|
97
203
|
|
|
98
204
|
|
|
@@ -101,6 +207,7 @@ to help you build a new value from other parts of the event.
|
|
|
101
207
|
|
|
102
208
|
* Value type is <<boolean,boolean>>
|
|
103
209
|
* Default value is `false`
|
|
210
|
+
* Option has no effect when _decoding_
|
|
104
211
|
|
|
105
212
|
Set to true to adhere to the specifications and encode using the CEF key name (short name) for the CEF field names.
|
|
106
213
|
|
|
@@ -109,8 +216,10 @@ Set to true to adhere to the specifications and encode using the CEF key name (s
|
|
|
109
216
|
|
|
110
217
|
* Value type is <<string,string>>
|
|
111
218
|
* Default value is `"6"`
|
|
219
|
+
* Option has no effect when _decoding_
|
|
112
220
|
|
|
113
|
-
|
|
221
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
222
|
+
value of the severity field in CEF header. The new value can include `%{foo}` strings
|
|
114
223
|
to help you build a new value from other parts of the event.
|
|
115
224
|
|
|
116
225
|
Defined as field of type string to allow sprintf. The value will be validated
|
|
@@ -122,8 +231,10 @@ All invalid values will be mapped to the default of 6.
|
|
|
122
231
|
|
|
123
232
|
* Value type is <<string,string>>
|
|
124
233
|
* Default value is `"Logstash"`
|
|
234
|
+
* Option has no effect when _decoding_
|
|
125
235
|
|
|
126
|
-
|
|
236
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
237
|
+
value of the signature ID field in CEF header. The new value can include `%{foo}` strings
|
|
127
238
|
to help you build a new value from other parts of the event.
|
|
128
239
|
|
|
129
240
|
[id="plugins-{type}s-{plugin}-vendor"]
|
|
@@ -131,8 +242,10 @@ to help you build a new value from other parts of the event.
|
|
|
131
242
|
|
|
132
243
|
* Value type is <<string,string>>
|
|
133
244
|
* Default value is `"Elasticsearch"`
|
|
245
|
+
* Option has no effect when _decoding_
|
|
134
246
|
|
|
135
|
-
|
|
247
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
248
|
+
value of the device vendor field in CEF header. The new value can include `%{foo}` strings
|
|
136
249
|
to help you build a new value from other parts of the event.
|
|
137
250
|
|
|
138
251
|
[id="plugins-{type}s-{plugin}-version"]
|
|
@@ -140,8 +253,9 @@ to help you build a new value from other parts of the event.
|
|
|
140
253
|
|
|
141
254
|
* Value type is <<string,string>>
|
|
142
255
|
* Default value is `"1.0"`
|
|
256
|
+
* Option has no effect when _decoding_
|
|
143
257
|
|
|
144
|
-
|
|
258
|
+
When this codec is used in an Output Plugin, this option can be used to specify the
|
|
259
|
+
value of the device version field in CEF header. The new value can include `%{foo}` strings
|
|
145
260
|
to help you build a new value from other parts of the event.
|
|
146
261
|
|
|
147
|
-
|
data/lib/logstash/codecs/cef.rb
CHANGED
|
@@ -3,6 +3,9 @@ require "logstash/util/buftok"
|
|
|
3
3
|
require "logstash/util/charset"
|
|
4
4
|
require "logstash/codecs/base"
|
|
5
5
|
require "json"
|
|
6
|
+
require "time"
|
|
7
|
+
|
|
8
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
|
6
9
|
|
|
7
10
|
# Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
|
8
11
|
# Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
|
@@ -13,6 +16,10 @@ require "json"
|
|
|
13
16
|
class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
14
17
|
config_name "cef"
|
|
15
18
|
|
|
19
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
|
|
20
|
+
|
|
21
|
+
InvalidTimestamp = Class.new(StandardError)
|
|
22
|
+
|
|
16
23
|
# Device vendor field in CEF header. The new value can include `%{foo}` strings
|
|
17
24
|
# to help you build a new value from other parts of the event.
|
|
18
25
|
config :vendor, :validate => :string, :default => "Elasticsearch"
|
|
@@ -68,106 +75,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
68
75
|
# * `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
|
69
76
|
config :delimiter, :validate => :string
|
|
70
77
|
|
|
78
|
+
# When parsing timestamps that do not include a UTC offset in payloads that do not
|
|
79
|
+
# include the device's timezone, the default timezone is used.
|
|
80
|
+
# If none is provided the system timezone is used.
|
|
81
|
+
config :default_timezone, :validate => :string
|
|
82
|
+
|
|
83
|
+
# The locale is used to parse abbreviated month names from some CEF timestamp
|
|
84
|
+
# formats.
|
|
85
|
+
# If none is provided, the system default is used.
|
|
86
|
+
config :locale, :validate => :string
|
|
87
|
+
|
|
71
88
|
# If raw_data_field is set, during decode of an event an additional field with
|
|
72
89
|
# the provided name is added, which contains the raw data.
|
|
73
90
|
config :raw_data_field, :validate => :string
|
|
74
91
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
#
|
|
78
|
-
|
|
79
|
-
"act" => "deviceAction",
|
|
80
|
-
"app" => "applicationProtocol",
|
|
81
|
-
"c6a1" => "deviceCustomIPv6Address1",
|
|
82
|
-
"c6a1Label" => "deviceCustomIPv6Address1Label",
|
|
83
|
-
"c6a2" => "deviceCustomIPv6Address2",
|
|
84
|
-
"c6a2Label" => "deviceCustomIPv6Address2Label",
|
|
85
|
-
"c6a3" => "deviceCustomIPv6Address3",
|
|
86
|
-
"c6a3Label" => "deviceCustomIPv6Address3Label",
|
|
87
|
-
"c6a4" => "deviceCustomIPv6Address4",
|
|
88
|
-
"c6a4Label" => "deviceCustomIPv6Address4Label",
|
|
89
|
-
"cat" => "deviceEventCategory",
|
|
90
|
-
"cfp1" => "deviceCustomFloatingPoint1",
|
|
91
|
-
"cfp1Label" => "deviceCustomFloatingPoint1Label",
|
|
92
|
-
"cfp2" => "deviceCustomFloatingPoint2",
|
|
93
|
-
"cfp2Label" => "deviceCustomFloatingPoint2Label",
|
|
94
|
-
"cfp3" => "deviceCustomFloatingPoint3",
|
|
95
|
-
"cfp3Label" => "deviceCustomFloatingPoint3Label",
|
|
96
|
-
"cfp4" => "deviceCustomFloatingPoint4",
|
|
97
|
-
"cfp4Label" => "deviceCustomFloatingPoint4Label",
|
|
98
|
-
"cn1" => "deviceCustomNumber1",
|
|
99
|
-
"cn1Label" => "deviceCustomNumber1Label",
|
|
100
|
-
"cn2" => "deviceCustomNumber2",
|
|
101
|
-
"cn2Label" => "deviceCustomNumber2Label",
|
|
102
|
-
"cn3" => "deviceCustomNumber3",
|
|
103
|
-
"cn3Label" => "deviceCustomNumber3Label",
|
|
104
|
-
"cnt" => "baseEventCount",
|
|
105
|
-
"cs1" => "deviceCustomString1",
|
|
106
|
-
"cs1Label" => "deviceCustomString1Label",
|
|
107
|
-
"cs2" => "deviceCustomString2",
|
|
108
|
-
"cs2Label" => "deviceCustomString2Label",
|
|
109
|
-
"cs3" => "deviceCustomString3",
|
|
110
|
-
"cs3Label" => "deviceCustomString3Label",
|
|
111
|
-
"cs4" => "deviceCustomString4",
|
|
112
|
-
"cs4Label" => "deviceCustomString4Label",
|
|
113
|
-
"cs5" => "deviceCustomString5",
|
|
114
|
-
"cs5Label" => "deviceCustomString5Label",
|
|
115
|
-
"cs6" => "deviceCustomString6",
|
|
116
|
-
"cs6Label" => "deviceCustomString6Label",
|
|
117
|
-
"dhost" => "destinationHostName",
|
|
118
|
-
"dmac" => "destinationMacAddress",
|
|
119
|
-
"dntdom" => "destinationNtDomain",
|
|
120
|
-
"dpid" => "destinationProcessId",
|
|
121
|
-
"dpriv" => "destinationUserPrivileges",
|
|
122
|
-
"dproc" => "destinationProcessName",
|
|
123
|
-
"dpt" => "destinationPort",
|
|
124
|
-
"dst" => "destinationAddress",
|
|
125
|
-
"duid" => "destinationUserId",
|
|
126
|
-
"duser" => "destinationUserName",
|
|
127
|
-
"dvc" => "deviceAddress",
|
|
128
|
-
"dvchost" => "deviceHostName",
|
|
129
|
-
"dvcpid" => "deviceProcessId",
|
|
130
|
-
"end" => "endTime",
|
|
131
|
-
"fname" => "fileName",
|
|
132
|
-
"fsize" => "fileSize",
|
|
133
|
-
"in" => "bytesIn",
|
|
134
|
-
"msg" => "message",
|
|
135
|
-
"out" => "bytesOut",
|
|
136
|
-
"outcome" => "eventOutcome",
|
|
137
|
-
"proto" => "transportProtocol",
|
|
138
|
-
"request" => "requestUrl",
|
|
139
|
-
"rt" => "deviceReceiptTime",
|
|
140
|
-
"shost" => "sourceHostName",
|
|
141
|
-
"smac" => "sourceMacAddress",
|
|
142
|
-
"sntdom" => "sourceNtDomain",
|
|
143
|
-
"spid" => "sourceProcessId",
|
|
144
|
-
"spriv" => "sourceUserPrivileges",
|
|
145
|
-
"sproc" => "sourceProcessName",
|
|
146
|
-
"spt" => "sourcePort",
|
|
147
|
-
"src" => "sourceAddress",
|
|
148
|
-
"start" => "startTime",
|
|
149
|
-
"suid" => "sourceUserId",
|
|
150
|
-
"suser" => "sourceUserName",
|
|
151
|
-
"ahost" => "agentHostName",
|
|
152
|
-
"art" => "agentReceiptTime",
|
|
153
|
-
"at" => "agentType",
|
|
154
|
-
"aid" => "agentId",
|
|
155
|
-
"_cefVer" => "cefVersion",
|
|
156
|
-
"agt" => "agentAddress",
|
|
157
|
-
"av" => "agentVersion",
|
|
158
|
-
"atz" => "agentTimeZone",
|
|
159
|
-
"dtz" => "destinationTimeZone",
|
|
160
|
-
"slong" => "sourceLongitude",
|
|
161
|
-
"slat" => "sourceLatitude",
|
|
162
|
-
"dlong" => "destinationLongitude",
|
|
163
|
-
"dlat" => "destinationLatitude",
|
|
164
|
-
"catdt" => "categoryDeviceType",
|
|
165
|
-
"mrt" => "managerReceiptTime",
|
|
166
|
-
"amac" => "agentMacAddress"
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
# Reverse mapping of CEF full field names to CEF extensions field names for encoding into a CEF event for output.
|
|
170
|
-
REVERSE_MAPPINGS = MAPPINGS.invert
|
|
92
|
+
# Defines whether a set of device-specific CEF fields represent the _observer_,
|
|
93
|
+
# or the actual `host` on which the event occurred. If this codec handles a mix,
|
|
94
|
+
# it is safe to use the default `observer`.
|
|
95
|
+
config :device, :validate => %w(observer host), :default => 'observer'
|
|
171
96
|
|
|
172
97
|
# A CEF Header is a sequence of zero or more:
|
|
173
98
|
# - backslash-escaped pipes; OR
|
|
@@ -255,6 +180,12 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
255
180
|
@delimiter = @delimiter.gsub("\\r", "\r").gsub("\\n", "\n")
|
|
256
181
|
@buffer = FileWatch::BufferedTokenizer.new(@delimiter)
|
|
257
182
|
end
|
|
183
|
+
|
|
184
|
+
require_relative 'cef/timestamp_normalizer'
|
|
185
|
+
@timestamp_normalzer = TimestampNormalizer.new(locale: @locale, timezone: @default_timezone)
|
|
186
|
+
|
|
187
|
+
generate_header_fields!
|
|
188
|
+
generate_mappings!
|
|
258
189
|
end
|
|
259
190
|
|
|
260
191
|
public
|
|
@@ -286,7 +217,7 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
286
217
|
|
|
287
218
|
# Use a scanning parser to capture the HEADER_FIELDS
|
|
288
219
|
unprocessed_data = data
|
|
289
|
-
|
|
220
|
+
@header_fields.each do |field_name|
|
|
290
221
|
match_data = HEADER_SCANNER.match(unprocessed_data)
|
|
291
222
|
break if match_data.nil? # missing fields
|
|
292
223
|
|
|
@@ -304,22 +235,24 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
304
235
|
message = unprocessed_data
|
|
305
236
|
|
|
306
237
|
# Try and parse out the syslog header if there is one
|
|
307
|
-
|
|
238
|
+
cef_version_field = @header_fields[0]
|
|
239
|
+
if (cef_version = event.get(cef_version_field)).include?(' ')
|
|
308
240
|
split_cef_version = cef_version.rpartition(' ')
|
|
309
|
-
event.set(
|
|
310
|
-
event.set(
|
|
241
|
+
event.set(@syslog_header, split_cef_version[0])
|
|
242
|
+
event.set(cef_version_field, split_cef_version[2])
|
|
311
243
|
end
|
|
312
244
|
|
|
313
245
|
# Get rid of the CEF bit in the version
|
|
314
|
-
event.set(
|
|
246
|
+
event.set(cef_version_field, delete_cef_prefix(event.get(cef_version_field)))
|
|
315
247
|
|
|
316
248
|
# Use a scanning parser to capture the Extension Key/Value Pairs
|
|
317
249
|
if message && message.include?('=')
|
|
318
250
|
message = message.strip
|
|
251
|
+
extension_fields = {}
|
|
319
252
|
|
|
320
253
|
message.scan(EXTENSION_KEY_VALUE_SCANNER) do |extension_field_key, raw_extension_field_value|
|
|
321
254
|
# expand abbreviated extension field keys
|
|
322
|
-
extension_field_key =
|
|
255
|
+
extension_field_key = @decode_mapping.fetch(extension_field_key, extension_field_key)
|
|
323
256
|
|
|
324
257
|
# convert extension field name to strict legal field_reference, fixing field names with ambiguous array-like syntax
|
|
325
258
|
extension_field_key = extension_field_key.sub(EXTENSION_KEY_ARRAY_CAPTURE, '[\1]\2') if extension_field_key.end_with?(']')
|
|
@@ -327,7 +260,21 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
327
260
|
# process legal extension field value escapes
|
|
328
261
|
extension_field_value = raw_extension_field_value.gsub(EXTENSION_VALUE_ESCAPE_CAPTURE, '\1')
|
|
329
262
|
|
|
330
|
-
|
|
263
|
+
extension_fields[extension_field_key] = extension_field_value
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# in ECS mode, normalize timestamps including timezone.
|
|
267
|
+
if ecs_compatibility != :disabled
|
|
268
|
+
device_timezone = extension_fields['[event][timezone]']
|
|
269
|
+
@timestamp_fields.each do |timestamp_field_name|
|
|
270
|
+
raw_timestamp = extension_fields.delete(timestamp_field_name) or next
|
|
271
|
+
value = normalize_timestamp(raw_timestamp, device_timezone)
|
|
272
|
+
event.set(timestamp_field_name, value)
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
extension_fields.each do |field_key, field_value|
|
|
277
|
+
event.set(field_key, field_value)
|
|
331
278
|
end
|
|
332
279
|
end
|
|
333
280
|
|
|
@@ -368,6 +315,234 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
368
315
|
|
|
369
316
|
private
|
|
370
317
|
|
|
318
|
+
def generate_header_fields!
|
|
319
|
+
# @header_fields is an _ordered_ set of fields.
|
|
320
|
+
@header_fields = [
|
|
321
|
+
ecs_select[disabled: 'cefVersion', v1: '[cef][version]'],
|
|
322
|
+
ecs_select[disabled: 'deviceVendor', v1: '[observer][vendor]'],
|
|
323
|
+
ecs_select[disabled: 'deviceProduct', v1: '[observer][product]'],
|
|
324
|
+
ecs_select[disabled: 'deviceVersion', v1: '[observer][version]'],
|
|
325
|
+
ecs_select[disabled: 'deviceEventClassId', v1: '[event][code]'],
|
|
326
|
+
ecs_select[disabled: 'name', v1: '[cef][name]'],
|
|
327
|
+
ecs_select[disabled: 'severity', v1: '[event][severity]']
|
|
328
|
+
].map(&:freeze).freeze
|
|
329
|
+
# the @syslog_header is the field name used when a syslog header preceeds the CEF Version.
|
|
330
|
+
@syslog_header = ecs_select[disabled:'syslog',v1:'[log][syslog][header]']
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
class CEFField
|
|
334
|
+
##
|
|
335
|
+
# @param name [String]: the full CEF name of a field
|
|
336
|
+
# @param key [String] (optional): an abbreviated CEF key to use when encoding a value with `reverse_mapping => true`
|
|
337
|
+
# when left unspecified, the `key` is the field's `name`.
|
|
338
|
+
# @param ecs_field [String] (optional): an ECS-compatible field reference to use, with square-bracket syntax.
|
|
339
|
+
# when left unspecified, the `ecs_field` is the field's `name`.
|
|
340
|
+
# @param legacy [String] (optional): a legacy CEF name to support in pass-through.
|
|
341
|
+
# in decoding mode without ECS, field name will be used as-provided.
|
|
342
|
+
# in encoding mode without ECS when provided to `fields` and `reverse_mapping => false`,
|
|
343
|
+
# field name will be used as-provided.
|
|
344
|
+
# @param priority [Integer] (optional): when multiple fields resolve to the same ECS field name, the field with the
|
|
345
|
+
# highest `prioriry` will be used by the encoder.
|
|
346
|
+
def initialize(name, key: name, ecs_field: name, legacy:nil, priority:0, normalize:nil)
|
|
347
|
+
@name = name
|
|
348
|
+
@key = key
|
|
349
|
+
@ecs_field = ecs_field
|
|
350
|
+
@legacy = legacy
|
|
351
|
+
@priority = priority
|
|
352
|
+
@normalize = normalize
|
|
353
|
+
end
|
|
354
|
+
attr_reader :name
|
|
355
|
+
attr_reader :key
|
|
356
|
+
attr_reader :ecs_field
|
|
357
|
+
attr_reader :legacy
|
|
358
|
+
attr_reader :priority
|
|
359
|
+
attr_reader :normalize
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
def generate_mappings!
|
|
363
|
+
encode_mapping = Hash.new
|
|
364
|
+
decode_mapping = Hash.new
|
|
365
|
+
timestamp_fields = Set.new
|
|
366
|
+
[
|
|
367
|
+
CEFField.new("agentAddress", key: "agt", ecs_field: "[agent][ip]"),
|
|
368
|
+
CEFField.new("agentDnsDomain", ecs_field: "[cef][agent][registered_domain]", priority: 10),
|
|
369
|
+
CEFField.new("agentHostName", key: "ahost", ecs_field: "[agent][name]"),
|
|
370
|
+
CEFField.new("agentId", key: "aid", ecs_field: "[agent][id]"),
|
|
371
|
+
CEFField.new("agentMacAddress", key: "amac", ecs_field: "[agent][mac]"),
|
|
372
|
+
CEFField.new("agentNtDomain", ecs_field: "[cef][agent][registered_domain]"),
|
|
373
|
+
CEFField.new("agentReceiptTime", key: "art", ecs_field: "[event][created]", normalize: :timestamp),
|
|
374
|
+
CEFField.new("agentTimeZone", key: "atz", ecs_field: "[cef][agent][timezone]"),
|
|
375
|
+
CEFField.new("agentTranslatedAddress", ecs_field: "[cef][agent][nat][ip]"),
|
|
376
|
+
CEFField.new("agentTranslatedZoneExternalID", ecs_field: "[cef][agent][translated_zone][external_id]"),
|
|
377
|
+
CEFField.new("agentTranslatedZoneURI", ecs_field: "[cef][agent][translated_zone][uri]"),
|
|
378
|
+
CEFField.new("agentType", key: "at", ecs_field: "[agent][type]"),
|
|
379
|
+
CEFField.new("agentVersion", key: "av", ecs_field: "[agent][version]"),
|
|
380
|
+
CEFField.new("agentZoneExternalID", ecs_field: "[cef][agent][zone][external_id]"),
|
|
381
|
+
CEFField.new("agentZoneURI", ecs_field: "[cef][agent][zone][uri]"),
|
|
382
|
+
CEFField.new("applicationProtocol", key: "app", ecs_field: "[network][protocol]"),
|
|
383
|
+
CEFField.new("baseEventCount", key: "cnt", ecs_field: "[cef][base_event_count]"),
|
|
384
|
+
CEFField.new("bytesIn", key: "in", ecs_field: "[source][bytes]"),
|
|
385
|
+
CEFField.new("bytesOut", key: "out", ecs_field: "[destination][bytes]"),
|
|
386
|
+
CEFField.new("categoryDeviceType", key: "catdt", ecs_field: "[cef][device_type]"),
|
|
387
|
+
CEFField.new("customerExternalID", ecs_field: "[organization][id]"),
|
|
388
|
+
CEFField.new("customerURI", ecs_field: "[organization][name]"),
|
|
389
|
+
CEFField.new("destinationAddress", key: "dst", ecs_field: "[destination][ip]"),
|
|
390
|
+
CEFField.new("destinationDnsDomain", ecs_field: "[destination][registered_domain]", priority: 10),
|
|
391
|
+
CEFField.new("destinationGeoLatitude", key: "dlat", ecs_field: "[destination][geo][location][lat]", legacy: "destinationLatitude"),
|
|
392
|
+
CEFField.new("destinationGeoLongitude", key: "dlong", ecs_field: "[destination][geo][location][lon]", legacy: "destinationLongitude"),
|
|
393
|
+
CEFField.new("destinationHostName", key: "dhost", ecs_field: "[destination][domain]"),
|
|
394
|
+
CEFField.new("destinationMacAddress", key: "dmac", ecs_field: "[destination][mac]"),
|
|
395
|
+
CEFField.new("destinationNtDomain", key: "dntdom", ecs_field: "[destination][registered_domain]"),
|
|
396
|
+
CEFField.new("destinationPort", key: "dpt", ecs_field: "[destination][port]"),
|
|
397
|
+
CEFField.new("destinationProcessId", key: "dpid", ecs_field: "[destination][process][pid]"),
|
|
398
|
+
CEFField.new("destinationProcessName", key: "dproc", ecs_field: "[destination][process][name]"),
|
|
399
|
+
CEFField.new("destinationServiceName", ecs_field: "[destination][service][name]"),
|
|
400
|
+
CEFField.new("destinationTranslatedAddress", ecs_field: "[destination][nat][ip]"),
|
|
401
|
+
CEFField.new("destinationTranslatedPort", ecs_field: "[destination][nat][port]"),
|
|
402
|
+
CEFField.new("destinationTranslatedZoneExternalID", ecs_field: "[cef][destination][translated_zone][external_id]"),
|
|
403
|
+
CEFField.new("destinationTranslatedZoneURI", ecs_field: "[cef][destination][translated_zone][uri]"),
|
|
404
|
+
CEFField.new("destinationUserId", key: "duid", ecs_field: "[destination][user][id]"),
|
|
405
|
+
CEFField.new("destinationUserName", key: "duser", ecs_field: "[destination][user][name]"),
|
|
406
|
+
CEFField.new("destinationUserPrivileges", key: "dpriv", ecs_field: "[destination][user][group][name]"),
|
|
407
|
+
CEFField.new("destinationZoneExternalID", ecs_field: "[cef][destination][zone][external_id]"),
|
|
408
|
+
CEFField.new("destinationZoneURI", ecs_field: "[cef][destination][zone][uri]"),
|
|
409
|
+
CEFField.new("deviceAction", key: "act", ecs_field: "[event][action]"),
|
|
410
|
+
CEFField.new("deviceAddress", key: "dvc", ecs_field: "[#{@device}][ip]"),
|
|
411
|
+
CEFField.new("deviceCustomFloatingPoint1", key: "cfp1", ecs_field: "[cef][device_custom_floating_point_1][value]"),
|
|
412
|
+
CEFField.new("deviceCustomFloatingPoint1Label", key: "cfp1Label", ecs_field: "[cef][device_custom_floating_point_1][label]"),
|
|
413
|
+
CEFField.new("deviceCustomFloatingPoint2", key: "cfp2", ecs_field: "[cef][device_custom_floating_point_2][value]"),
|
|
414
|
+
CEFField.new("deviceCustomFloatingPoint2Label", key: "cfp2Label", ecs_field: "[cef][device_custom_floating_point_2][label]"),
|
|
415
|
+
CEFField.new("deviceCustomFloatingPoint3", key: "cfp3", ecs_field: "[cef][device_custom_floating_point_3][value]"),
|
|
416
|
+
CEFField.new("deviceCustomFloatingPoint3Label", key: "cfp3Label", ecs_field: "[cef][device_custom_floating_point_3][label]"),
|
|
417
|
+
CEFField.new("deviceCustomFloatingPoint4", key: "cfp4", ecs_field: "[cef][device_custom_floating_point_4][value]"),
|
|
418
|
+
CEFField.new("deviceCustomFloatingPoint4Label", key: "cfp4Label", ecs_field: "[cef][device_custom_floating_point_4][label]"),
|
|
419
|
+
CEFField.new("deviceCustomIPv6Address1", key: "c6a1", ecs_field: "[cef][device_custom_ipv6_address_1][value]"),
|
|
420
|
+
CEFField.new("deviceCustomIPv6Address1Label", key: "c6a1Label", ecs_field: "[cef][device_custom_ipv6_address_1][label]"),
|
|
421
|
+
CEFField.new("deviceCustomIPv6Address2", key: "c6a2", ecs_field: "[cef][device_custom_ipv6_address_2][value]"),
|
|
422
|
+
CEFField.new("deviceCustomIPv6Address2Label", key: "c6a2Label", ecs_field: "[cef][device_custom_ipv6_address_2][label]"),
|
|
423
|
+
CEFField.new("deviceCustomIPv6Address3", key: "c6a3", ecs_field: "[cef][device_custom_ipv6_address_3][value]"),
|
|
424
|
+
CEFField.new("deviceCustomIPv6Address3Label", key: "c6a3Label", ecs_field: "[cef][device_custom_ipv6_address_3][label]"),
|
|
425
|
+
CEFField.new("deviceCustomIPv6Address4", key: "c6a4", ecs_field: "[cef][device_custom_ipv6_address_4][value]"),
|
|
426
|
+
CEFField.new("deviceCustomIPv6Address4Label", key: "c6a4Label", ecs_field: "[cef][device_custom_ipv6_address_4][label]"),
|
|
427
|
+
CEFField.new("deviceCustomNumber1", key: "cn1", ecs_field: "[cef][device_custom_number_1][value]"),
|
|
428
|
+
CEFField.new("deviceCustomNumber1Label", key: "cn1Label", ecs_field: "[cef][device_custom_number_1][label]"),
|
|
429
|
+
CEFField.new("deviceCustomNumber2", key: "cn2", ecs_field: "[cef][device_custom_number_2][value]"),
|
|
430
|
+
CEFField.new("deviceCustomNumber2Label", key: "cn2Label", ecs_field: "[cef][device_custom_number_2][label]"),
|
|
431
|
+
CEFField.new("deviceCustomNumber3", key: "cn3", ecs_field: "[cef][device_custom_number_3][value]"),
|
|
432
|
+
CEFField.new("deviceCustomNumber3Label", key: "cn3Label", ecs_field: "[cef][device_custom_number_3][label]"),
|
|
433
|
+
CEFField.new("deviceCustomString1", key: "cs1", ecs_field: "[cef][device_custom_string_1][value]"),
|
|
434
|
+
CEFField.new("deviceCustomString1Label", key: "cs1Label", ecs_field: "[cef][device_custom_string_1][label]"),
|
|
435
|
+
CEFField.new("deviceCustomString2", key: "cs2", ecs_field: "[cef][device_custom_string_2][value]"),
|
|
436
|
+
CEFField.new("deviceCustomString2Label", key: "cs2Label", ecs_field: "[cef][device_custom_string_2][label]"),
|
|
437
|
+
CEFField.new("deviceCustomString3", key: "cs3", ecs_field: "[cef][device_custom_string_3][value]"),
|
|
438
|
+
CEFField.new("deviceCustomString3Label", key: "cs3Label", ecs_field: "[cef][device_custom_string_3][label]"),
|
|
439
|
+
CEFField.new("deviceCustomString4", key: "cs4", ecs_field: "[cef][device_custom_string_4][value]"),
|
|
440
|
+
CEFField.new("deviceCustomString4Label", key: "cs4Label", ecs_field: "[cef][device_custom_string_4][label]"),
|
|
441
|
+
CEFField.new("deviceCustomString5", key: "cs5", ecs_field: "[cef][device_custom_string_5][value]"),
|
|
442
|
+
CEFField.new("deviceCustomString5Label", key: "cs5Label", ecs_field: "[cef][device_custom_string_5][label]"),
|
|
443
|
+
CEFField.new("deviceCustomString6", key: "cs6", ecs_field: "[cef][device_custom_string_6][value]"),
|
|
444
|
+
CEFField.new("deviceCustomString6Label", key: "cs6Label", ecs_field: "[cef][device_custom_string_6][label]"),
|
|
445
|
+
CEFField.new("deviceDirection", ecs_field: "[network][direction]"),
|
|
446
|
+
CEFField.new("deviceDnsDomain", ecs_field: "[#{@device}][registered_domain]", priority: 10),
|
|
447
|
+
CEFField.new("deviceEventCategory", key: "cat", ecs_field: "[cef][category]"),
|
|
448
|
+
CEFField.new("deviceExternalId", ecs_field: (@device == 'host' ? "[host][id]" : "[observer][name]")),
|
|
449
|
+
CEFField.new("deviceFacility", ecs_field: "[log][syslog][facility][code]"),
|
|
450
|
+
CEFField.new("deviceHostName", key: "dvchost", ecs_field: (@device == 'host' ? '[host][name]' : '[observer][hostname]')),
|
|
451
|
+
CEFField.new("deviceInboundInterface", ecs_field: "[observer][ingress][interface][name]"),
|
|
452
|
+
CEFField.new("deviceMacAddress", key: "dvcmac", ecs_field: "[@device][mac]"),
|
|
453
|
+
CEFField.new("deviceNtDomain", ecs_field: "[cef][nt_domain]"),
|
|
454
|
+
CEFField.new("deviceOutboundInterface", ecs_field: "[observer][egress][interface][name]"),
|
|
455
|
+
CEFField.new("devicePayloadId", ecs_field: "[cef][payload_id]"),
|
|
456
|
+
CEFField.new("deviceProcessId", key: "dvcpid", ecs_field: "[process][pid]"),
|
|
457
|
+
CEFField.new("deviceProcessName", ecs_field: "[process][name]"),
|
|
458
|
+
CEFField.new("deviceReceiptTime", key: "rt", ecs_field: "@timestamp", normalize: :timestamp),
|
|
459
|
+
CEFField.new("deviceTimeZone", key: "dtz", ecs_field: "[event][timezone]", legacy: "destinationTimeZone"),
|
|
460
|
+
CEFField.new("deviceTranslatedAddress", ecs_field: "[host][nat][ip]"),
|
|
461
|
+
CEFField.new("deviceTranslatedZoneExternalID", ecs_field: "[cef][translated_zone][external_id]"),
|
|
462
|
+
CEFField.new("deviceTranslatedZoneURI", ecs_field: "[cef][translated_zone][uri]"),
|
|
463
|
+
CEFField.new("deviceVersion", ecs_field: "[observer][version]"),
|
|
464
|
+
CEFField.new("deviceZoneExternalID", ecs_field: "[cef][zone][external_id]"),
|
|
465
|
+
CEFField.new("deviceZoneURI", ecs_field: "[cef][zone][uri]"),
|
|
466
|
+
CEFField.new("endTime", key: "end", ecs_field: "[event][end]", normalize: :timestamp),
|
|
467
|
+
CEFField.new("eventId", ecs_field: "[event][id]"),
|
|
468
|
+
CEFField.new("eventOutcome", key: "outcome", ecs_field: "[event][outcome]"),
|
|
469
|
+
CEFField.new("externalId", ecs_field: "[cef][external_id]"),
|
|
470
|
+
CEFField.new("fileCreateTime", ecs_field: "[file][created]"),
|
|
471
|
+
CEFField.new("fileHash", ecs_field: "[file][hash]]"),
|
|
472
|
+
CEFField.new("fileId", ecs_field: "[file][inode]"),
|
|
473
|
+
CEFField.new("fileModificationTime", ecs_field: "[file][mtime]", normalize: :timestamp),
|
|
474
|
+
CEFField.new("fileName", key: "fname", ecs_field: "[file][name]"),
|
|
475
|
+
CEFField.new("filePath", ecs_field: "[file][path]"),
|
|
476
|
+
CEFField.new("filePermission", ecs_field: "[file][group]"),
|
|
477
|
+
CEFField.new("fileSize", key: "fsize", ecs_field: "[file][size]"),
|
|
478
|
+
CEFField.new("fileType", ecs_field: "[file][extension]"),
|
|
479
|
+
CEFField.new("managerReceiptTime", key: "mrt", ecs_field: "[event][ingested]", normalize: :timestamp),
|
|
480
|
+
CEFField.new("message", key: "msg", ecs_field: "[message]"),
|
|
481
|
+
CEFField.new("oldFileCreateTime", ecs_field: "[cef][old_file][created]", normalize: :timestamp),
|
|
482
|
+
CEFField.new("oldFileHash", ecs_field: "[cef][old_file][hash]"),
|
|
483
|
+
CEFField.new("oldFileId", ecs_field: "[cef][old_file][inode]"),
|
|
484
|
+
CEFField.new("oldFileModificationTime", ecs_field: "[cef][old_file][mtime]", normalize: :timestamp),
|
|
485
|
+
CEFField.new("oldFileName", ecs_field: "[cef][old_file][name]"),
|
|
486
|
+
CEFField.new("oldFilePath", ecs_field: "[cef][old_file][path]"),
|
|
487
|
+
CEFField.new("oldFilePermission", ecs_field: "[cef][old_file][group]"),
|
|
488
|
+
CEFField.new("oldFileSize", ecs_field: "[cef][old_file][size]"),
|
|
489
|
+
CEFField.new("oldFileType", ecs_field: "[cef][old_file][extension]"),
|
|
490
|
+
CEFField.new("rawEvent", ecs_field: "[event][original]"),
|
|
491
|
+
CEFField.new("Reason", key: "reason", ecs_field: "[event][reason]"),
|
|
492
|
+
CEFField.new("requestClientApplication", ecs_field: "[user_agent][original]"),
|
|
493
|
+
CEFField.new("requestContext", ecs_field: "[http][request][referrer]"),
|
|
494
|
+
CEFField.new("requestCookies", ecs_field: "[cef][request][cookies]"),
|
|
495
|
+
CEFField.new("requestMethod", ecs_field: "[http][request][method]"),
|
|
496
|
+
CEFField.new("requestUrl", key: "request", ecs_field: "[url][original]"),
|
|
497
|
+
CEFField.new("sourceAddress", key: "src", ecs_field: "[source][ip]"),
|
|
498
|
+
CEFField.new("sourceDnsDomain", ecs_field: "[source][registered_domain]", priority: 10),
|
|
499
|
+
CEFField.new("sourceGeoLatitude", key: "slat", ecs_field: "[source][geo][location][lat]", legacy: "sourceLatitude"),
|
|
500
|
+
CEFField.new("sourceGeoLongitude", key: "slong", ecs_field: "[source][geo][location][lon]", legacy: "sourceLongitude"),
|
|
501
|
+
CEFField.new("sourceHostName", key: "shost", ecs_field: "[source][domain]"),
|
|
502
|
+
CEFField.new("sourceMacAddress", key: "smac", ecs_field: "[source][mac]"),
|
|
503
|
+
CEFField.new("sourceNtDomain", key: "sntdom", ecs_field: "[source][registered_domain]"),
|
|
504
|
+
CEFField.new("sourcePort", key: "spt", ecs_field: "[source][port]"),
|
|
505
|
+
CEFField.new("sourceProcessId", key: "spid", ecs_field: "[source][process][pid]"),
|
|
506
|
+
CEFField.new("sourceProcessName", key: "sproc", ecs_field: "[source][process][name]"),
|
|
507
|
+
CEFField.new("sourceServiceName", ecs_field: "[source][service][name]"),
|
|
508
|
+
CEFField.new("sourceTranslatedAddress", ecs_field: "[source][nat][ip]"),
|
|
509
|
+
CEFField.new("sourceTranslatedPort", ecs_field: "[source][nat][port]"),
|
|
510
|
+
CEFField.new("sourceTranslatedZoneExternalID", ecs_field: "[cef][source][translated_zone][external_id]"),
|
|
511
|
+
CEFField.new("sourceTranslatedZoneURI", ecs_field: "[cef][source][translated_zone][uri]"),
|
|
512
|
+
CEFField.new("sourceUserId", key: "suid", ecs_field: "[source][user][id]"),
|
|
513
|
+
CEFField.new("sourceUserName", key: "suser", ecs_field: "[source][user][name]"),
|
|
514
|
+
CEFField.new("sourceUserPrivileges", key: "spriv", ecs_field: "[source][user][group][name]"),
|
|
515
|
+
CEFField.new("sourceZoneExternalID", ecs_field: "[cef][source][zone][external_id]"),
|
|
516
|
+
CEFField.new("sourceZoneURI", ecs_field: "[cef][source][zone][uri]"),
|
|
517
|
+
CEFField.new("startTime", key: "start", ecs_field: "[event][start]", normalize: :timestamp),
|
|
518
|
+
CEFField.new("transportProtocol", key: "proto", ecs_field: "[network][transport]"),
|
|
519
|
+
CEFField.new("type", ecs_field: "[cef][type]"),
|
|
520
|
+
].sort_by(&:priority).each do |cef|
|
|
521
|
+
field_name = ecs_select[disabled:cef.name, v1:cef.ecs_field]
|
|
522
|
+
|
|
523
|
+
# whether the source is a cef_key or cef_name, normalize to field_name
|
|
524
|
+
decode_mapping[cef.key] = field_name
|
|
525
|
+
decode_mapping[cef.name] = field_name
|
|
526
|
+
|
|
527
|
+
# whether source is a cef_name or a field_name, normalize to target
|
|
528
|
+
normalized_encode_target = @reverse_mapping ? cef.key : cef.name
|
|
529
|
+
encode_mapping[field_name] = normalized_encode_target
|
|
530
|
+
encode_mapping[cef.name] = normalized_encode_target unless cef.name == field_name
|
|
531
|
+
|
|
532
|
+
# if a field has an alias, normalize pass-through
|
|
533
|
+
if cef.legacy
|
|
534
|
+
decode_mapping[cef.legacy] = ecs_select[disabled:cef.legacy, v1:cef.ecs_field]
|
|
535
|
+
encode_mapping[cef.legacy] = @reverse_mapping ? cef.key : cef.legacy
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
timestamp_fields << field_name if ecs_compatibility != :disabled && cef.normalize == :timestamp
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
@decode_mapping = decode_mapping.dup.freeze
|
|
542
|
+
@encode_mapping = encode_mapping.dup.freeze
|
|
543
|
+
@timestamp_fields = timestamp_fields.dup.freeze
|
|
544
|
+
end
|
|
545
|
+
|
|
371
546
|
# Escape pipes and backslashes in the header. Equal signs are ok.
|
|
372
547
|
# Newlines are forbidden.
|
|
373
548
|
def sanitize_header_field(value)
|
|
@@ -392,17 +567,23 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
|
392
567
|
.gsub(EXTENSION_VALUE_SANITIZER_PATTERN, EXTENSION_VALUE_SANITIZER_MAPPING)
|
|
393
568
|
end
|
|
394
569
|
|
|
570
|
+
def normalize_timestamp(value, device_timezone_name)
|
|
571
|
+
value = @timestamp_normalzer.normalize(value, device_timezone_name).iso8601(9)
|
|
572
|
+
|
|
573
|
+
LogStash::Timestamp.new(value)
|
|
574
|
+
rescue => e
|
|
575
|
+
@logger.error("Failed to parse CEF timestamp value `#{value}` (#{e.message})")
|
|
576
|
+
raise InvalidTimestamp.new("Not a valid CEF timestamp: `#{value}`")
|
|
577
|
+
end
|
|
578
|
+
|
|
395
579
|
def get_value(fieldname, event)
|
|
396
580
|
val = event.get(fieldname)
|
|
397
581
|
|
|
398
582
|
return nil if val.nil?
|
|
399
583
|
|
|
400
|
-
key =
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
key = REVERSE_MAPPINGS[key] || key
|
|
404
|
-
end
|
|
405
|
-
|
|
584
|
+
key = @encode_mapping.fetch(fieldname, fieldname)
|
|
585
|
+
key = sanitize_extension_key(key)
|
|
586
|
+
|
|
406
587
|
case val
|
|
407
588
|
when Array, Hash
|
|
408
589
|
return "#{key}=#{sanitize_extension_val(val.to_json)}"
|