logstash-codec-cef 4.1.2-java → 4.1.3-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +10 -1
- data/docs/index.asciidoc +164 -0
- data/lib/logstash/codecs/cef.rb +9 -3
- data/logstash-codec-cef.gemspec +2 -2
- data/spec/codecs/cef_spec.rb +14 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: f5ad55d1f43500d6329f21ed2943bf811f0750869c4c895828909eff71d91c16
|
4
|
+
data.tar.gz: 543d21d49806ba3c99cd6f14061762f22ca74876f3326c581ad0964754a43822
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ededdd96038a9039ffe29f524411c19de38f64672ea32b340ed067c59e682225ff9649ca789d9cd4eef599732413ef5a30af65fd376c16f369498a47cbdda255
|
7
|
+
data.tar.gz: 2b859e530dcf3f9b0f99bab1e3f8a2fa81762c35a291c788288d5a19260c71158ecab9f00a4814b3f67bb92cc75ff12ff91e11ba897ac5c5d6239400bcb2aee3
|
data/Gemfile
CHANGED
@@ -1,2 +1,11 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
|
-
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/docs/index.asciidoc
ADDED
@@ -0,0 +1,164 @@
|
|
1
|
+
:plugin: cef
|
2
|
+
:type: codec
|
3
|
+
|
4
|
+
///////////////////////////////////////////
|
5
|
+
START - GENERATED VARIABLES, DO NOT EDIT!
|
6
|
+
///////////////////////////////////////////
|
7
|
+
:version: %VERSION%
|
8
|
+
:release_date: %RELEASE_DATE%
|
9
|
+
:changelog_url: %CHANGELOG_URL%
|
10
|
+
:include_path: ../../../../logstash/docs/include
|
11
|
+
///////////////////////////////////////////
|
12
|
+
END - GENERATED VARIABLES, DO NOT EDIT!
|
13
|
+
///////////////////////////////////////////
|
14
|
+
|
15
|
+
[id="plugins-{type}-{plugin}"]
|
16
|
+
|
17
|
+
=== Cef codec plugin
|
18
|
+
|
19
|
+
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
|
21
|
+
==== Description
|
22
|
+
|
23
|
+
Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
24
|
+
Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
25
|
+
https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
|
26
|
+
|
27
|
+
If this codec receives a payload from an input that is not a valid CEF message, then it will
|
28
|
+
produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
29
|
+
|
30
|
+
[id="plugins-{type}s-{plugin}-options"]
|
31
|
+
==== Cef Codec Configuration Options
|
32
|
+
|
33
|
+
[cols="<,<,<",options="header",]
|
34
|
+
|=======================================================================
|
35
|
+
|Setting |Input type|Required
|
36
|
+
| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
|
37
|
+
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
38
|
+
| <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
|
39
|
+
| <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
|
40
|
+
| <<plugins-{type}s-{plugin}-severity>> |<<string,string>>|No
|
41
|
+
| <<plugins-{type}s-{plugin}-signature>> |<<string,string>>|No
|
42
|
+
| <<plugins-{type}s-{plugin}-vendor>> |<<string,string>>|No
|
43
|
+
| <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
|
44
|
+
|=======================================================================
|
45
|
+
|
46
|
+
|
47
|
+
|
48
|
+
[id="plugins-{type}s-{plugin}-delimiter"]
|
49
|
+
===== `delimiter`
|
50
|
+
|
51
|
+
* Value type is <<string,string>>
|
52
|
+
* There is no default value for this setting.
|
53
|
+
|
54
|
+
If your input puts a delimiter between each CEF event, you'll want to set
|
55
|
+
this to be that delimiter.
|
56
|
+
|
57
|
+
For example, with the TCP input, you probably want to put this:
|
58
|
+
|
59
|
+
input {
|
60
|
+
tcp {
|
61
|
+
codec => cef { delimiter => "\r\n" }
|
62
|
+
# ...
|
63
|
+
}
|
64
|
+
}
|
65
|
+
|
66
|
+
This setting allows the following character sequences to have special meaning:
|
67
|
+
|
68
|
+
* `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
|
69
|
+
* `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
70
|
+
|
71
|
+
[id="plugins-{type}s-{plugin}-deprecated_v1_fields"]
|
72
|
+
===== `deprecated_v1_fields` (DEPRECATED)
|
73
|
+
|
74
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
75
|
+
* Value type is <<boolean,boolean>>
|
76
|
+
* There is no default value for this setting.
|
77
|
+
|
78
|
+
Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase
|
79
|
+
the index size and data stored in outputs like Elasticsearch
|
80
|
+
This option is available to ease transition to new schema
|
81
|
+
|
82
|
+
[id="plugins-{type}s-{plugin}-fields"]
|
83
|
+
===== `fields`
|
84
|
+
|
85
|
+
* Value type is <<array,array>>
|
86
|
+
* Default value is `[]`
|
87
|
+
|
88
|
+
Fields to be included in CEV extension part as key/value pairs
|
89
|
+
|
90
|
+
[id="plugins-{type}s-{plugin}-name"]
|
91
|
+
===== `name`
|
92
|
+
|
93
|
+
* Value type is <<string,string>>
|
94
|
+
* Default value is `"Logstash"`
|
95
|
+
|
96
|
+
Name field in CEF header. The new value can include `%{foo}` strings
|
97
|
+
to help you build a new value from other parts of the event.
|
98
|
+
|
99
|
+
[id="plugins-{type}s-{plugin}-product"]
|
100
|
+
===== `product`
|
101
|
+
|
102
|
+
* Value type is <<string,string>>
|
103
|
+
* Default value is `"Logstash"`
|
104
|
+
|
105
|
+
Device product field in CEF header. The new value can include `%{foo}` strings
|
106
|
+
to help you build a new value from other parts of the event.
|
107
|
+
|
108
|
+
[id="plugins-{type}s-{plugin}-sev"]
|
109
|
+
===== `sev` (DEPRECATED)
|
110
|
+
|
111
|
+
* DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
|
112
|
+
* Value type is <<string,string>>
|
113
|
+
* There is no default value for this setting.
|
114
|
+
|
115
|
+
Deprecated severity field for CEF header. The new value can include `%{foo}` strings
|
116
|
+
to help you build a new value from other parts of the event.
|
117
|
+
|
118
|
+
This field is used only if :severity is unchanged set to the default value.
|
119
|
+
|
120
|
+
Defined as field of type string to allow sprintf. The value will be validated
|
121
|
+
to be an integer in the range from 0 to 10 (including).
|
122
|
+
All invalid values will be mapped to the default of 6.
|
123
|
+
|
124
|
+
[id="plugins-{type}s-{plugin}-severity"]
|
125
|
+
===== `severity`
|
126
|
+
|
127
|
+
* Value type is <<string,string>>
|
128
|
+
* Default value is `"6"`
|
129
|
+
|
130
|
+
Severity field in CEF header. The new value can include `%{foo}` strings
|
131
|
+
to help you build a new value from other parts of the event.
|
132
|
+
|
133
|
+
Defined as field of type string to allow sprintf. The value will be validated
|
134
|
+
to be an integer in the range from 0 to 10 (including).
|
135
|
+
All invalid values will be mapped to the default of 6.
|
136
|
+
|
137
|
+
[id="plugins-{type}s-{plugin}-signature"]
|
138
|
+
===== `signature`
|
139
|
+
|
140
|
+
* Value type is <<string,string>>
|
141
|
+
* Default value is `"Logstash"`
|
142
|
+
|
143
|
+
Signature ID field in CEF header. The new value can include `%{foo}` strings
|
144
|
+
to help you build a new value from other parts of the event.
|
145
|
+
|
146
|
+
[id="plugins-{type}s-{plugin}-vendor"]
|
147
|
+
===== `vendor`
|
148
|
+
|
149
|
+
* Value type is <<string,string>>
|
150
|
+
* Default value is `"Elasticsearch"`
|
151
|
+
|
152
|
+
Device vendor field in CEF header. The new value can include `%{foo}` strings
|
153
|
+
to help you build a new value from other parts of the event.
|
154
|
+
|
155
|
+
[id="plugins-{type}s-{plugin}-version"]
|
156
|
+
===== `version`
|
157
|
+
|
158
|
+
* Value type is <<string,string>>
|
159
|
+
* Default value is `"1.0"`
|
160
|
+
|
161
|
+
Device version field in CEF header. The new value can include `%{foo}` strings
|
162
|
+
to help you build a new value from other parts of the event.
|
163
|
+
|
164
|
+
|
data/lib/logstash/codecs/cef.rb
CHANGED
@@ -5,7 +5,7 @@ require "json"
|
|
5
5
|
|
6
6
|
# Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
|
7
7
|
# Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
|
8
|
-
# https://
|
8
|
+
# https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
|
9
9
|
#
|
10
10
|
# If this codec receives a payload from an input that is not a valid CEF message, then it will
|
11
11
|
# produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
|
@@ -76,10 +76,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
76
76
|
# * `\\n` (backslash "n") - means newline (ASCII 0x0A)
|
77
77
|
config :delimiter, :validate => :string
|
78
78
|
|
79
|
+
# If raw_data_field is set, during decode of an event an additional field with
|
80
|
+
# the provided name is added, which contains the raw data.
|
81
|
+
config :raw_data_field, :validate => :string
|
82
|
+
|
79
83
|
HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
|
80
84
|
|
81
85
|
# Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
|
82
|
-
MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "
|
86
|
+
MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2Label", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint3Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNtDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime", "amac" => "agentMacAddress" }
|
83
87
|
|
84
88
|
DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
|
85
89
|
|
@@ -113,11 +117,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
|
|
113
117
|
end
|
114
118
|
|
115
119
|
def handle(data, &block)
|
120
|
+
event = LogStash::Event.new
|
121
|
+
event.set(raw_data_field, data) unless raw_data_field.nil?
|
122
|
+
|
116
123
|
# Strip any quotations at the start and end, flex connectors seem to send this
|
117
124
|
if data[0] == "\""
|
118
125
|
data = data[1..-2]
|
119
126
|
end
|
120
|
-
event = LogStash::Event.new
|
121
127
|
|
122
128
|
# Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping
|
123
129
|
# The better solution for the splitting regex would be /(?<!\\(\\\\)*)[\|]/, but this
|
data/logstash-codec-cef.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-cef'
|
4
|
-
s.version = '4.1.
|
4
|
+
s.version = '4.1.3'
|
5
5
|
s.platform = 'java'
|
6
6
|
s.licenses = ['Apache License (2.0)']
|
7
7
|
s.summary = "CEF codec to parse and encode CEF formated logs"
|
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
|
14
14
|
# Files
|
15
|
-
s.files = Dir[
|
15
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
16
16
|
|
17
17
|
# Tests
|
18
18
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
data/spec/codecs/cef_spec.rb
CHANGED
@@ -533,7 +533,7 @@ describe LogStash::Codecs::CEF do
|
|
533
533
|
end
|
534
534
|
end
|
535
535
|
|
536
|
-
let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success'}
|
536
|
+
let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
|
537
537
|
it "should translate most known abbreviated CEF field names" do
|
538
538
|
subject.decode(translate_abbreviated_cef_fields) do |e|
|
539
539
|
validate(e)
|
@@ -545,6 +545,7 @@ describe LogStash::Codecs::CEF do
|
|
545
545
|
insist { e.get("sourcePort") } == "11024"
|
546
546
|
insist { e.get("destinationPort") } == "9200"
|
547
547
|
insist { e.get("eventOutcome") } == "Success"
|
548
|
+
insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
|
548
549
|
end
|
549
550
|
end
|
550
551
|
|
@@ -555,6 +556,18 @@ describe LogStash::Codecs::CEF do
|
|
555
556
|
insist { e.get('syslog') } == 'Syslogdate Sysloghost'
|
556
557
|
end
|
557
558
|
end
|
559
|
+
|
560
|
+
context "with raw_data_field set" do
|
561
|
+
subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
|
562
|
+
|
563
|
+
it "should return the raw message in field message_raw" do
|
564
|
+
subject.decode(message) do |e|
|
565
|
+
validate(e)
|
566
|
+
insist { e.get("message_raw") } == message
|
567
|
+
end
|
568
|
+
end
|
569
|
+
end
|
570
|
+
|
558
571
|
end
|
559
572
|
|
560
573
|
context "decode with deprecated version option" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-cef
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.1.
|
4
|
+
version: 4.1.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,7 +44,9 @@ dependencies:
|
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: '0'
|
47
|
-
description: This gem is a Logstash plugin required to be installed on top of the
|
47
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
48
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
49
|
+
gem is not a stand-alone program
|
48
50
|
email: info@elastic.co
|
49
51
|
executables: []
|
50
52
|
extensions: []
|
@@ -56,6 +58,7 @@ files:
|
|
56
58
|
- LICENSE
|
57
59
|
- NOTICE.TXT
|
58
60
|
- README.md
|
61
|
+
- docs/index.asciidoc
|
59
62
|
- lib/logstash/codecs/cef.rb
|
60
63
|
- logstash-codec-cef.gemspec
|
61
64
|
- spec/codecs/cef_spec.rb
|
@@ -81,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
81
84
|
version: '0'
|
82
85
|
requirements: []
|
83
86
|
rubyforge_project:
|
84
|
-
rubygems_version: 2.
|
87
|
+
rubygems_version: 2.6.11
|
85
88
|
signing_key:
|
86
89
|
specification_version: 4
|
87
90
|
summary: CEF codec to parse and encode CEF formated logs
|