logstash-codec-cef 4.1.2-java → 4.1.3-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 4970c9754e3a998c7768dfd91127908d00754c2d
4
- data.tar.gz: c89d21a60488b04b8f305346b35715bf398b22b2
2
+ SHA256:
3
+ metadata.gz: f5ad55d1f43500d6329f21ed2943bf811f0750869c4c895828909eff71d91c16
4
+ data.tar.gz: 543d21d49806ba3c99cd6f14061762f22ca74876f3326c581ad0964754a43822
5
5
  SHA512:
6
- metadata.gz: 1ddba453c75fec89baf0c374663721f20df6edfa054f1ff8b904e75cd95da4db92d1d6bd20771de5d1f6ba8d4bae1e7a5f15068cc250568a718265129a5b9802
7
- data.tar.gz: e1132893ea01846719b888fe1f83c0d79bd1f7c6a1600ccbd6c6f6aa996915069b8eeb6f4bfae90aeb66d7f3f7281fef031bba8df4d161a98685eb6dab143923
6
+ metadata.gz: ededdd96038a9039ffe29f524411c19de38f64672ea32b340ed067c59e682225ff9649ca789d9cd4eef599732413ef5a30af65fd376c16f369498a47cbdda255
7
+ data.tar.gz: 2b859e530dcf3f9b0f99bab1e3f8a2fa81762c35a291c788288d5a19260c71158ecab9f00a4814b3f67bb92cc75ff12ff91e11ba897ac5c5d6239400bcb2aee3
data/Gemfile CHANGED
@@ -1,2 +1,11 @@
1
1
  source 'https://rubygems.org'
2
- gemspec
2
+
3
+ gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,164 @@
1
+ :plugin: cef
2
+ :type: codec
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Cef codec plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
24
+ Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
25
+ https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
26
+
27
+ If this codec receives a payload from an input that is not a valid CEF message, then it will
28
+ produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
29
+
30
+ [id="plugins-{type}s-{plugin}-options"]
31
+ ==== Cef Codec Configuration Options
32
+
33
+ [cols="<,<,<",options="header",]
34
+ |=======================================================================
35
+ |Setting |Input type|Required
36
+ | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
37
+ | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
38
+ | <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
39
+ | <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
40
+ | <<plugins-{type}s-{plugin}-severity>> |<<string,string>>|No
41
+ | <<plugins-{type}s-{plugin}-signature>> |<<string,string>>|No
42
+ | <<plugins-{type}s-{plugin}-vendor>> |<<string,string>>|No
43
+ | <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
44
+ |=======================================================================
45
+
46
+ &nbsp;
47
+
48
+ [id="plugins-{type}s-{plugin}-delimiter"]
49
+ ===== `delimiter`
50
+
51
+ * Value type is <<string,string>>
52
+ * There is no default value for this setting.
53
+
54
+ If your input puts a delimiter between each CEF event, you'll want to set
55
+ this to be that delimiter.
56
+
57
+ For example, with the TCP input, you probably want to put this:
58
+
59
+ input {
60
+ tcp {
61
+ codec => cef { delimiter => "\r\n" }
62
+ # ...
63
+ }
64
+ }
65
+
66
+ This setting allows the following character sequences to have special meaning:
67
+
68
+ * `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
69
+ * `\\n` (backslash "n") - means newline (ASCII 0x0A)
70
+
71
+ [id="plugins-{type}s-{plugin}-deprecated_v1_fields"]
72
+ ===== `deprecated_v1_fields` (DEPRECATED)
73
+
74
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
75
+ * Value type is <<boolean,boolean>>
76
+ * There is no default value for this setting.
77
+
78
+ Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase
79
+ the index size and data stored in outputs like Elasticsearch
80
+ This option is available to ease transition to new schema
81
+
82
+ [id="plugins-{type}s-{plugin}-fields"]
83
+ ===== `fields`
84
+
85
+ * Value type is <<array,array>>
86
+ * Default value is `[]`
87
+
88
+ Fields to be included in CEV extension part as key/value pairs
89
+
90
+ [id="plugins-{type}s-{plugin}-name"]
91
+ ===== `name`
92
+
93
+ * Value type is <<string,string>>
94
+ * Default value is `"Logstash"`
95
+
96
+ Name field in CEF header. The new value can include `%{foo}` strings
97
+ to help you build a new value from other parts of the event.
98
+
99
+ [id="plugins-{type}s-{plugin}-product"]
100
+ ===== `product`
101
+
102
+ * Value type is <<string,string>>
103
+ * Default value is `"Logstash"`
104
+
105
+ Device product field in CEF header. The new value can include `%{foo}` strings
106
+ to help you build a new value from other parts of the event.
107
+
108
+ [id="plugins-{type}s-{plugin}-sev"]
109
+ ===== `sev` (DEPRECATED)
110
+
111
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
112
+ * Value type is <<string,string>>
113
+ * There is no default value for this setting.
114
+
115
+ Deprecated severity field for CEF header. The new value can include `%{foo}` strings
116
+ to help you build a new value from other parts of the event.
117
+
118
+ This field is used only if :severity is unchanged set to the default value.
119
+
120
+ Defined as field of type string to allow sprintf. The value will be validated
121
+ to be an integer in the range from 0 to 10 (including).
122
+ All invalid values will be mapped to the default of 6.
123
+
124
+ [id="plugins-{type}s-{plugin}-severity"]
125
+ ===== `severity`
126
+
127
+ * Value type is <<string,string>>
128
+ * Default value is `"6"`
129
+
130
+ Severity field in CEF header. The new value can include `%{foo}` strings
131
+ to help you build a new value from other parts of the event.
132
+
133
+ Defined as field of type string to allow sprintf. The value will be validated
134
+ to be an integer in the range from 0 to 10 (including).
135
+ All invalid values will be mapped to the default of 6.
136
+
137
+ [id="plugins-{type}s-{plugin}-signature"]
138
+ ===== `signature`
139
+
140
+ * Value type is <<string,string>>
141
+ * Default value is `"Logstash"`
142
+
143
+ Signature ID field in CEF header. The new value can include `%{foo}` strings
144
+ to help you build a new value from other parts of the event.
145
+
146
+ [id="plugins-{type}s-{plugin}-vendor"]
147
+ ===== `vendor`
148
+
149
+ * Value type is <<string,string>>
150
+ * Default value is `"Elasticsearch"`
151
+
152
+ Device vendor field in CEF header. The new value can include `%{foo}` strings
153
+ to help you build a new value from other parts of the event.
154
+
155
+ [id="plugins-{type}s-{plugin}-version"]
156
+ ===== `version`
157
+
158
+ * Value type is <<string,string>>
159
+ * Default value is `"1.0"`
160
+
161
+ Device version field in CEF header. The new value can include `%{foo}` strings
162
+ to help you build a new value from other parts of the event.
163
+
164
+
@@ -5,7 +5,7 @@ require "json"
5
5
 
6
6
  # Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
7
7
  # Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
8
- # https://protect724.hp.com/servlet/JiveServlet/downloadBody/1072-102-6-4697/CommonEventFormat.pdf
8
+ # https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
9
9
  #
10
10
  # If this codec receives a payload from an input that is not a valid CEF message, then it will
11
11
  # produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
@@ -76,10 +76,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
76
76
  # * `\\n` (backslash "n") - means newline (ASCII 0x0A)
77
77
  config :delimiter, :validate => :string
78
78
 
79
+ # If raw_data_field is set, during decode of an event an additional field with
80
+ # the provided name is added, which contains the raw data.
81
+ config :raw_data_field, :validate => :string
82
+
79
83
  HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
80
84
 
81
85
  # Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
82
- MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint4Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNTDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime" }
86
+ MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2Label", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint3Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNtDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime", "amac" => "agentMacAddress" }
83
87
 
84
88
  DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
85
89
 
@@ -113,11 +117,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
113
117
  end
114
118
 
115
119
  def handle(data, &block)
120
+ event = LogStash::Event.new
121
+ event.set(raw_data_field, data) unless raw_data_field.nil?
122
+
116
123
  # Strip any quotations at the start and end, flex connectors seem to send this
117
124
  if data[0] == "\""
118
125
  data = data[1..-2]
119
126
  end
120
- event = LogStash::Event.new
121
127
 
122
128
  # Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping
123
129
  # The better solution for the splitting regex would be /(?<!\\(\\\\)*)[\|]/, but this
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-cef'
4
- s.version = '4.1.2'
4
+ s.version = '4.1.3'
5
5
  s.platform = 'java'
6
6
  s.licenses = ['Apache License (2.0)']
7
7
  s.summary = "CEF codec to parse and encode CEF formated logs"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.require_paths = ["lib"]
13
13
 
14
14
  # Files
15
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
15
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
16
16
 
17
17
  # Tests
18
18
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -533,7 +533,7 @@ describe LogStash::Codecs::CEF do
533
533
  end
534
534
  end
535
535
 
536
- let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success'}
536
+ let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
537
537
  it "should translate most known abbreviated CEF field names" do
538
538
  subject.decode(translate_abbreviated_cef_fields) do |e|
539
539
  validate(e)
@@ -545,6 +545,7 @@ describe LogStash::Codecs::CEF do
545
545
  insist { e.get("sourcePort") } == "11024"
546
546
  insist { e.get("destinationPort") } == "9200"
547
547
  insist { e.get("eventOutcome") } == "Success"
548
+ insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
548
549
  end
549
550
  end
550
551
 
@@ -555,6 +556,18 @@ describe LogStash::Codecs::CEF do
555
556
  insist { e.get('syslog') } == 'Syslogdate Sysloghost'
556
557
  end
557
558
  end
559
+
560
+ context "with raw_data_field set" do
561
+ subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
562
+
563
+ it "should return the raw message in field message_raw" do
564
+ subject.decode(message) do |e|
565
+ validate(e)
566
+ insist { e.get("message_raw") } == message
567
+ end
568
+ end
569
+ end
570
+
558
571
  end
559
572
 
560
573
  context "decode with deprecated version option" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-cef
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.1.2
4
+ version: 4.1.3
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-26 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -44,7 +44,9 @@ dependencies:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
- description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
47
+ description: This gem is a Logstash plugin required to be installed on top of the
48
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
49
+ gem is not a stand-alone program
48
50
  email: info@elastic.co
49
51
  executables: []
50
52
  extensions: []
@@ -56,6 +58,7 @@ files:
56
58
  - LICENSE
57
59
  - NOTICE.TXT
58
60
  - README.md
61
+ - docs/index.asciidoc
59
62
  - lib/logstash/codecs/cef.rb
60
63
  - logstash-codec-cef.gemspec
61
64
  - spec/codecs/cef_spec.rb
@@ -81,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
81
84
  version: '0'
82
85
  requirements: []
83
86
  rubyforge_project:
84
- rubygems_version: 2.4.8
87
+ rubygems_version: 2.6.11
85
88
  signing_key:
86
89
  specification_version: 4
87
90
  summary: CEF codec to parse and encode CEF formated logs