logstash-codec-cef 4.1.2-java → 4.1.3-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile +10 -1
- data/docs/index.asciidoc +164 -0
- data/lib/logstash/codecs/cef.rb +9 -3
- data/logstash-codec-cef.gemspec +2 -2
- data/spec/codecs/cef_spec.rb +14 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: f5ad55d1f43500d6329f21ed2943bf811f0750869c4c895828909eff71d91c16
|
4
|
+
data.tar.gz: 543d21d49806ba3c99cd6f14061762f22ca74876f3326c581ad0964754a43822
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ededdd96038a9039ffe29f524411c19de38f64672ea32b340ed067c59e682225ff9649ca789d9cd4eef599732413ef5a30af65fd376c16f369498a47cbdda255
|
7
|
+
data.tar.gz: 2b859e530dcf3f9b0f99bab1e3f8a2fa81762c35a291c788288d5a19260c71158ecab9f00a4814b3f67bb92cc75ff12ff91e11ba897ac5c5d6239400bcb2aee3
|
data/Gemfile
CHANGED
@@ -1,2 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
-
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,164 @@
|
|
1
|
+
:plugin: cef
|
2
|
+
:type: codec
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Cef codec plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
24
|
+
Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
25
|
+
https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
|
26
|
+
|
27
|
+
If this codec receives a payload from an input that is not a valid CEF message, then it will
|
28
|
+
produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
29
|
+
|
30
|
+
[id="plugins-{type}s-{plugin}-options"]
|
31
|
+
==== Cef Codec Configuration Options
|
32
|
+
|
33
|
+
[cols="<,<,<",options="header",]
|
34
|
+
|=======================================================================
|
35
|
+
|Setting |Input type|Required
|
36
|
+
| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
|
37
|
+
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
38
|
+
| <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
|
39
|
+
| <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
|
40
|
+
| <<plugins-{type}s-{plugin}-severity>> |<<string,string>>|No
|
41
|
+
| <<plugins-{type}s-{plugin}-signature>> |<<string,string>>|No
|
42
|
+
| <<plugins-{type}s-{plugin}-vendor>> |<<string,string>>|No
|
43
|
+
| <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
|
44
|
+
|=======================================================================
|
45
|
+
|
46
|
+
|
47
|
+
|
48
|
+
[id="plugins-{type}s-{plugin}-delimiter"]
|
49
|
+
===== `delimiter`
|
50
|
+
|
51
|
+
* Value type is <<string,string>>
|
52
|
+
* There is no default value for this setting.
|
53
|
+
|
54
|
+
If your input puts a delimiter between each CEF event, you'll want to set
|
55
|
+
this to be that delimiter.
|
56
|
+
|
57
|
+
For example, with the TCP input, you probably want to put this:
|
58
|
+
|
59
|
+
input {
|
60
|
+
tcp {
|
61
|
+
codec => cef { delimiter => "\r\n" }
|
62
|
+
# ...
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
This setting allows the following character sequences to have special meaning:
|
67
|
+
|
68
|
+
* `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
|
69
|
+
* `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
70
|
+
|
71
|
+
[id="plugins-{type}s-{plugin}-deprecated_v1_fields"]
|
72
|
+
===== `deprecated_v1_fields` (DEPRECATED)
|
73
|
+
|
74
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
75
|
+
* Value type is <<boolean,boolean>>
|
76
|
+
* There is no default value for this setting.
|
77
|
+
|
78
|
+
Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase
|
79
|
+
the index size and data stored in outputs like Elasticsearch
|
80
|
+
This option is available to ease transition to new schema
|
81
|
+
|
82
|
+
[id="plugins-{type}s-{plugin}-fields"]
|
83
|
+
===== `fields`
|
84
|
+
|
85
|
+
* Value type is <<array,array>>
|
86
|
+
* Default value is `[]`
|
87
|
+
|
88
|
+
Fields to be included in CEV extension part as key/value pairs
|
89
|
+
|
90
|
+
[id="plugins-{type}s-{plugin}-name"]
|
91
|
+
===== `name`
|
92
|
+
|
93
|
+
* Value type is <<string,string>>
|
94
|
+
* Default value is `"Logstash"`
|
95
|
+
|
96
|
+
Name field in CEF header. The new value can include `%{foo}` strings
|
97
|
+
to help you build a new value from other parts of the event.
|
98
|
+
|
99
|
+
[id="plugins-{type}s-{plugin}-product"]
|
100
|
+
===== `product`
|
101
|
+
|
102
|
+
* Value type is <<string,string>>
|
103
|
+
* Default value is `"Logstash"`
|
104
|
+
|
105
|
+
Device product field in CEF header. The new value can include `%{foo}` strings
|
106
|
+
to help you build a new value from other parts of the event.
|
107
|
+
|
108
|
+
[id="plugins-{type}s-{plugin}-sev"]
|
109
|
+
===== `sev` (DEPRECATED)
|
110
|
+
|
111
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
112
|
+
* Value type is <<string,string>>
|
113
|
+
* There is no default value for this setting.
|
114
|
+
|
115
|
+
Deprecated severity field for CEF header. The new value can include `%{foo}` strings
|
116
|
+
to help you build a new value from other parts of the event.
|
117
|
+
|
118
|
+
This field is used only if :severity is unchanged set to the default value.
|
119
|
+
|
120
|
+
Defined as field of type string to allow sprintf. The value will be validated
|
121
|
+
to be an integer in the range from 0 to 10 (including).
|
122
|
+
All invalid values will be mapped to the default of 6.
|
123
|
+
|
124
|
+
[id="plugins-{type}s-{plugin}-severity"]
|
125
|
+
===== `severity`
|
126
|
+
|
127
|
+
* Value type is <<string,string>>
|
128
|
+
* Default value is `"6"`
|
129
|
+
|
130
|
+
Severity field in CEF header. The new value can include `%{foo}` strings
|
131
|
+
to help you build a new value from other parts of the event.
|
132
|
+
|
133
|
+
Defined as field of type string to allow sprintf. The value will be validated
|
134
|
+
to be an integer in the range from 0 to 10 (including).
|
135
|
+
All invalid values will be mapped to the default of 6.
|
136
|
+
|
137
|
+
[id="plugins-{type}s-{plugin}-signature"]
|
138
|
+
===== `signature`
|
139
|
+
|
140
|
+
* Value type is <<string,string>>
|
141
|
+
* Default value is `"Logstash"`
|
142
|
+
|
143
|
+
Signature ID field in CEF header. The new value can include `%{foo}` strings
|
144
|
+
to help you build a new value from other parts of the event.
|
145
|
+
|
146
|
+
[id="plugins-{type}s-{plugin}-vendor"]
|
147
|
+
===== `vendor`
|
148
|
+
|
149
|
+
* Value type is <<string,string>>
|
150
|
+
* Default value is `"Elasticsearch"`
|
151
|
+
|
152
|
+
Device vendor field in CEF header. The new value can include `%{foo}` strings
|
153
|
+
to help you build a new value from other parts of the event.
|
154
|
+
|
155
|
+
[id="plugins-{type}s-{plugin}-version"]
|
156
|
+
===== `version`
|
157
|
+
|
158
|
+
* Value type is <<string,string>>
|
159
|
+
* Default value is `"1.0"`
|
160
|
+
|
161
|
+
Device version field in CEF header. The new value can include `%{foo}` strings
|
162
|
+
to help you build a new value from other parts of the event.
|
163
|
+
|
164
|
+
|
data/lib/logstash/codecs/cef.rb
CHANGED
@@ -5,7 +5,7 @@ require "json"
|
|
5
5
|
|
6
6
|
# Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
7
7
|
# Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
8
|
-
# https://
|
8
|
+
# https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
|
9
9
|
#
|
10
10
|
# If this codec receives a payload from an input that is not a valid CEF message, then it will
|
11
11
|
# produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
@@ -76,10 +76,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
76
76
|
# * `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
77
77
|
config :delimiter, :validate => :string
|
78
78
|
|
79
|
+
# If raw_data_field is set, during decode of an event an additional field with
|
80
|
+
# the provided name is added, which contains the raw data.
|
81
|
+
config :raw_data_field, :validate => :string
|
82
|
+
|
79
83
|
HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
|
80
84
|
|
81
85
|
# Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
|
82
|
-
MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "
|
86
|
+
MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2Label", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint3Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNtDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime", "amac" => "agentMacAddress" }
|
83
87
|
|
84
88
|
DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
|
85
89
|
|
@@ -113,11 +117,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
113
117
|
end
|
114
118
|
|
115
119
|
def handle(data, &block)
|
120
|
+
event = LogStash::Event.new
|
121
|
+
event.set(raw_data_field, data) unless raw_data_field.nil?
|
122
|
+
|
116
123
|
# Strip any quotations at the start and end, flex connectors seem to send this
|
117
124
|
if data[0] == "\""
|
118
125
|
data = data[1..-2]
|
119
126
|
end
|
120
|
-
event = LogStash::Event.new
|
121
127
|
|
122
128
|
# Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping
|
123
129
|
# The better solution for the splitting regex would be /(?<!\\(\\\\)*)[\|]/, but this
|
data/logstash-codec-cef.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-cef'
|
4
|
-
s.version = '4.1.
|
4
|
+
s.version = '4.1.3'
|
5
5
|
s.platform = 'java'
|
6
6
|
s.licenses = ['Apache License (2.0)']
|
7
7
|
s.summary = "CEF codec to parse and encode CEF formated logs"
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
|
14
14
|
# Files
|
15
|
-
s.files = Dir[
|
15
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
16
16
|
|
17
17
|
# Tests
|
18
18
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
data/spec/codecs/cef_spec.rb
CHANGED
@@ -533,7 +533,7 @@ describe LogStash::Codecs::CEF do
|
|
533
533
|
end
|
534
534
|
end
|
535
535
|
|
536
|
-
let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success'}
|
536
|
+
let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
|
537
537
|
it "should translate most known abbreviated CEF field names" do
|
538
538
|
subject.decode(translate_abbreviated_cef_fields) do |e|
|
539
539
|
validate(e)
|
@@ -545,6 +545,7 @@ describe LogStash::Codecs::CEF do
|
|
545
545
|
insist { e.get("sourcePort") } == "11024"
|
546
546
|
insist { e.get("destinationPort") } == "9200"
|
547
547
|
insist { e.get("eventOutcome") } == "Success"
|
548
|
+
insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
|
548
549
|
end
|
549
550
|
end
|
550
551
|
|
@@ -555,6 +556,18 @@ describe LogStash::Codecs::CEF do
|
|
555
556
|
insist { e.get('syslog') } == 'Syslogdate Sysloghost'
|
556
557
|
end
|
557
558
|
end
|
559
|
+
|
560
|
+
context "with raw_data_field set" do
|
561
|
+
subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
|
562
|
+
|
563
|
+
it "should return the raw message in field message_raw" do
|
564
|
+
subject.decode(message) do |e|
|
565
|
+
validate(e)
|
566
|
+
insist { e.get("message_raw") } == message
|
567
|
+
end
|
568
|
+
end
|
569
|
+
end
|
570
|
+
|
558
571
|
end
|
559
572
|
|
560
573
|
context "decode with deprecated version option" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-cef
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.1.
|
4
|
+
version: 4.1.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,7 +44,9 @@ dependencies:
|
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
|
-
description: This gem is a Logstash plugin required to be installed on top of the
|
47
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
48
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
49
|
+
gem is not a stand-alone program
|
48
50
|
email: info@elastic.co
|
49
51
|
executables: []
|
50
52
|
extensions: []
|
@@ -56,6 +58,7 @@ files:
|
|
56
58
|
- LICENSE
|
57
59
|
- NOTICE.TXT
|
58
60
|
- README.md
|
61
|
+
- docs/index.asciidoc
|
59
62
|
- lib/logstash/codecs/cef.rb
|
60
63
|
- logstash-codec-cef.gemspec
|
61
64
|
- spec/codecs/cef_spec.rb
|
@@ -81,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
81
84
|
version: '0'
|
82
85
|
requirements: []
|
83
86
|
rubyforge_project:
|
84
|
-
rubygems_version: 2.
|
87
|
+
rubygems_version: 2.6.11
|
85
88
|
signing_key:
|
86
89
|
specification_version: 4
|
87
90
|
summary: CEF codec to parse and encode CEF formated logs
|