logstash-codec-cef 4.1.2-java → 4.1.3-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 4970c9754e3a998c7768dfd91127908d00754c2d
4
- data.tar.gz: c89d21a60488b04b8f305346b35715bf398b22b2
2
+ SHA256:
3
+ metadata.gz: f5ad55d1f43500d6329f21ed2943bf811f0750869c4c895828909eff71d91c16
4
+ data.tar.gz: 543d21d49806ba3c99cd6f14061762f22ca74876f3326c581ad0964754a43822
5
5
  SHA512:
6
- metadata.gz: 1ddba453c75fec89baf0c374663721f20df6edfa054f1ff8b904e75cd95da4db92d1d6bd20771de5d1f6ba8d4bae1e7a5f15068cc250568a718265129a5b9802
7
- data.tar.gz: e1132893ea01846719b888fe1f83c0d79bd1f7c6a1600ccbd6c6f6aa996915069b8eeb6f4bfae90aeb66d7f3f7281fef031bba8df4d161a98685eb6dab143923
6
+ metadata.gz: ededdd96038a9039ffe29f524411c19de38f64672ea32b340ed067c59e682225ff9649ca789d9cd4eef599732413ef5a30af65fd376c16f369498a47cbdda255
7
+ data.tar.gz: 2b859e530dcf3f9b0f99bab1e3f8a2fa81762c35a291c788288d5a19260c71158ecab9f00a4814b3f67bb92cc75ff12ff91e11ba897ac5c5d6239400bcb2aee3
data/Gemfile CHANGED
@@ -1,2 +1,11 @@
1
1
  source 'https://rubygems.org'
2
- gemspec
2
+
3
+ gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,164 @@
1
+ :plugin: cef
2
+ :type: codec
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Cef codec plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
24
+ Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
25
+ https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
26
+
27
+ If this codec receives a payload from an input that is not a valid CEF message, then it will
28
+ produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
29
+
30
+ [id="plugins-{type}s-{plugin}-options"]
31
+ ==== Cef Codec Configuration Options
32
+
33
+ [cols="<,<,<",options="header",]
34
+ |=======================================================================
35
+ |Setting |Input type|Required
36
+ | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
37
+ | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
38
+ | <<plugins-{type}s-{plugin}-name>> |<<string,string>>|No
39
+ | <<plugins-{type}s-{plugin}-product>> |<<string,string>>|No
40
+ | <<plugins-{type}s-{plugin}-severity>> |<<string,string>>|No
41
+ | <<plugins-{type}s-{plugin}-signature>> |<<string,string>>|No
42
+ | <<plugins-{type}s-{plugin}-vendor>> |<<string,string>>|No
43
+ | <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
44
+ |=======================================================================
45
+
46
+ &nbsp;
47
+
48
+ [id="plugins-{type}s-{plugin}-delimiter"]
49
+ ===== `delimiter`
50
+
51
+ * Value type is <<string,string>>
52
+ * There is no default value for this setting.
53
+
54
+ If your input puts a delimiter between each CEF event, you'll want to set
55
+ this to be that delimiter.
56
+
57
+ For example, with the TCP input, you probably want to put this:
58
+
59
+ input {
60
+ tcp {
61
+ codec => cef { delimiter => "\r\n" }
62
+ # ...
63
+ }
64
+ }
65
+
66
+ This setting allows the following character sequences to have special meaning:
67
+
68
+ * `\\r` (backslash "r") - means carriage return (ASCII 0x0D)
69
+ * `\\n` (backslash "n") - means newline (ASCII 0x0A)
70
+
71
+ [id="plugins-{type}s-{plugin}-deprecated_v1_fields"]
72
+ ===== `deprecated_v1_fields` (DEPRECATED)
73
+
74
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
75
+ * Value type is <<boolean,boolean>>
76
+ * There is no default value for this setting.
77
+
78
+ Set this flag if you want to have both v1 and v2 fields indexed at the same time. Note that this option will increase
79
+ the index size and data stored in outputs like Elasticsearch
80
+ This option is available to ease transition to new schema
81
+
82
+ [id="plugins-{type}s-{plugin}-fields"]
83
+ ===== `fields`
84
+
85
+ * Value type is <<array,array>>
86
+ * Default value is `[]`
87
+
88
+ Fields to be included in CEV extension part as key/value pairs
89
+
90
+ [id="plugins-{type}s-{plugin}-name"]
91
+ ===== `name`
92
+
93
+ * Value type is <<string,string>>
94
+ * Default value is `"Logstash"`
95
+
96
+ Name field in CEF header. The new value can include `%{foo}` strings
97
+ to help you build a new value from other parts of the event.
98
+
99
+ [id="plugins-{type}s-{plugin}-product"]
100
+ ===== `product`
101
+
102
+ * Value type is <<string,string>>
103
+ * Default value is `"Logstash"`
104
+
105
+ Device product field in CEF header. The new value can include `%{foo}` strings
106
+ to help you build a new value from other parts of the event.
107
+
108
+ [id="plugins-{type}s-{plugin}-sev"]
109
+ ===== `sev` (DEPRECATED)
110
+
111
+ * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
112
+ * Value type is <<string,string>>
113
+ * There is no default value for this setting.
114
+
115
+ Deprecated severity field for CEF header. The new value can include `%{foo}` strings
116
+ to help you build a new value from other parts of the event.
117
+
118
+ This field is used only if :severity is unchanged set to the default value.
119
+
120
+ Defined as field of type string to allow sprintf. The value will be validated
121
+ to be an integer in the range from 0 to 10 (including).
122
+ All invalid values will be mapped to the default of 6.
123
+
124
+ [id="plugins-{type}s-{plugin}-severity"]
125
+ ===== `severity`
126
+
127
+ * Value type is <<string,string>>
128
+ * Default value is `"6"`
129
+
130
+ Severity field in CEF header. The new value can include `%{foo}` strings
131
+ to help you build a new value from other parts of the event.
132
+
133
+ Defined as field of type string to allow sprintf. The value will be validated
134
+ to be an integer in the range from 0 to 10 (including).
135
+ All invalid values will be mapped to the default of 6.
136
+
137
+ [id="plugins-{type}s-{plugin}-signature"]
138
+ ===== `signature`
139
+
140
+ * Value type is <<string,string>>
141
+ * Default value is `"Logstash"`
142
+
143
+ Signature ID field in CEF header. The new value can include `%{foo}` strings
144
+ to help you build a new value from other parts of the event.
145
+
146
+ [id="plugins-{type}s-{plugin}-vendor"]
147
+ ===== `vendor`
148
+
149
+ * Value type is <<string,string>>
150
+ * Default value is `"Elasticsearch"`
151
+
152
+ Device vendor field in CEF header. The new value can include `%{foo}` strings
153
+ to help you build a new value from other parts of the event.
154
+
155
+ [id="plugins-{type}s-{plugin}-version"]
156
+ ===== `version`
157
+
158
+ * Value type is <<string,string>>
159
+ * Default value is `"1.0"`
160
+
161
+ Device version field in CEF header. The new value can include `%{foo}` strings
162
+ to help you build a new value from other parts of the event.
163
+
164
+
@@ -5,7 +5,7 @@ require "json"
5
5
 
6
6
  # Implementation of a Logstash codec for the ArcSight Common Event Format (CEF)
7
7
  # Based on Revision 20 of Implementing ArcSight CEF, dated from June 05, 2013
8
- # https://protect724.hp.com/servlet/JiveServlet/downloadBody/1072-102-6-4697/CommonEventFormat.pdf
8
+ # https://community.saas.hpe.com/dcvta86296/attachments/dcvta86296/connector-documentation/1116/1/CommonEventFormatv23.pdf
9
9
  #
10
10
  # If this codec receives a payload from an input that is not a valid CEF message, then it will
11
11
  # produce an event with the payload as the 'message' field and a '_cefparsefailure' tag.
@@ -76,10 +76,14 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
76
76
  # * `\\n` (backslash "n") - means newline (ASCII 0x0A)
77
77
  config :delimiter, :validate => :string
78
78
 
79
+ # If raw_data_field is set, during decode of an event an additional field with
80
+ # the provided name is added, which contains the raw data.
81
+ config :raw_data_field, :validate => :string
82
+
79
83
  HEADER_FIELDS = ['cefVersion','deviceVendor','deviceProduct','deviceVersion','deviceEventClassId','name','severity']
80
84
 
81
85
  # Translating and flattening the CEF extensions with known field names as documented in the Common Event Format whitepaper
82
- MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint4Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNTDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime" }
86
+ MAPPINGS = { "act" => "deviceAction", "app" => "applicationProtocol", "c6a1" => "deviceCustomIPv6Address1", "c6a1Label" => "deviceCustomIPv6Address1Label", "c6a2" => "deviceCustomIPv6Address2", "c6a2Label" => "deviceCustomIPv6Address2Label", "c6a3" => "deviceCustomIPv6Address3", "c6a3Label" => "deviceCustomIPv6Address3Label", "c6a4" => "deviceCustomIPv6Address4", "c6a4Label" => "deviceCustomIPv6Address4Label", "cat" => "deviceEventCategory", "cfp1" => "deviceCustomFloatingPoint1", "cfp1Label" => "deviceCustomFloatingPoint1Label", "cfp2" => "deviceCustomFloatingPoint2", "cfp2Label" => "deviceCustomFloatingPoint2Label", "cfp3" => "deviceCustomFloatingPoint3", "cfp3Label" => "deviceCustomFloatingPoint3Label", "cfp4" => "deviceCustomFloatingPoint4", "cfp4Label" => "deviceCustomFloatingPoint4Label", "cn1" => "deviceCustomNumber1", "cn1Label" => "deviceCustomNumber1Label", "cn2" => "deviceCustomNumber2", "cn2Label" => "deviceCustomNumber2Label", "cn3" => "deviceCustomNumber3", "cn3Label" => "deviceCustomNumber3Label", "cnt" => "baseEventCount", "cs1" => "deviceCustomString1", "cs1Label" => "deviceCustomString1Label", "cs2" => "deviceCustomString2", "cs2Label" => "deviceCustomString2Label", "cs3" => "deviceCustomString3", "cs3Label" => "deviceCustomString3Label", "cs4" => "deviceCustomString4", "cs4Label" => "deviceCustomString4Label", "cs5" => "deviceCustomString5", "cs5Label" => "deviceCustomString5Label", "cs6" => "deviceCustomString6", "cs6Label" => "deviceCustomString6Label", "dhost" => "destinationHostName", "dmac" => "destinationMacAddress", "dntdom" => "destinationNtDomain", "dpid" => "destinationProcessId", "dpriv" => "destinationUserPrivileges", "dproc" => "destinationProcessName", "dpt" => "destinationPort", "dst" => "destinationAddress", "duid" => "destinationUserId", "duser" => "destinationUserName", "dvc" => "deviceAddress", "dvchost" => "deviceHostName", "dvcpid" => "deviceProcessId", "end" => "endTime", "fname" => "fileName", "fsize" => "fileSize", "in" => "bytesIn", "msg" => "message", "out" => "bytesOut", "outcome" => "eventOutcome", "proto" => "transportProtocol", "request" => "requestUrl", "rt" => "deviceReceiptTime", "shost" => "sourceHostName", "smac" => "sourceMacAddress", "sntdom" => "sourceNtDomain", "spid" => "sourceProcessId", "spriv" => "sourceUserPrivileges", "sproc" => "sourceProcessName", "spt" => "sourcePort", "src" => "sourceAddress", "start" => "startTime", "suid" => "sourceUserId", "suser" => "sourceUserName", "ahost" => "agentHost", "art" => "agentReceiptTime", "at" => "agentType", "aid" => "agentId", "_cefVer" => "cefVersion", "agt" => "agentAddress", "av" => "agentVersion", "atz" => "agentTimeZone", "dtz" => "destinationTimeZone", "slong" => "sourceLongitude", "slat" => "sourceLatitude", "dlong" => "destinationLongitude", "dlat" => "destinationLatitude", "catdt" => "categoryDeviceType", "mrt" => "managerReceiptTime", "amac" => "agentMacAddress" }
83
87
 
84
88
  DEPRECATED_HEADER_FIELDS = ['cef_version','cef_vendor','cef_product','cef_device_version','cef_sigid','cef_name','cef_severity']
85
89
 
@@ -113,11 +117,13 @@ class LogStash::Codecs::CEF < LogStash::Codecs::Base
113
117
  end
114
118
 
115
119
  def handle(data, &block)
120
+ event = LogStash::Event.new
121
+ event.set(raw_data_field, data) unless raw_data_field.nil?
122
+
116
123
  # Strip any quotations at the start and end, flex connectors seem to send this
117
124
  if data[0] == "\""
118
125
  data = data[1..-2]
119
126
  end
120
- event = LogStash::Event.new
121
127
 
122
128
  # Split by the pipes, pipes in the extension part are perfectly valid and do not need escaping
123
129
  # The better solution for the splitting regex would be /(?<!\\(\\\\)*)[\|]/, but this
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-cef'
4
- s.version = '4.1.2'
4
+ s.version = '4.1.3'
5
5
  s.platform = 'java'
6
6
  s.licenses = ['Apache License (2.0)']
7
7
  s.summary = "CEF codec to parse and encode CEF formated logs"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.require_paths = ["lib"]
13
13
 
14
14
  # Files
15
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
15
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
16
16
 
17
17
  # Tests
18
18
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -533,7 +533,7 @@ describe LogStash::Codecs::CEF do
533
533
  end
534
534
  end
535
535
 
536
- let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success'}
536
+ let (:translate_abbreviated_cef_fields) {'CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 proto=TCP shost=source.host.name dhost=destination.host.name spt=11024 dpt=9200 outcome=Success amac=00:80:48:1c:24:91'}
537
537
  it "should translate most known abbreviated CEF field names" do
538
538
  subject.decode(translate_abbreviated_cef_fields) do |e|
539
539
  validate(e)
@@ -545,6 +545,7 @@ describe LogStash::Codecs::CEF do
545
545
  insist { e.get("sourcePort") } == "11024"
546
546
  insist { e.get("destinationPort") } == "9200"
547
547
  insist { e.get("eventOutcome") } == "Success"
548
+ insist { e.get("agentMacAddress")} == "00:80:48:1c:24:91"
548
549
  end
549
550
  end
550
551
 
@@ -555,6 +556,18 @@ describe LogStash::Codecs::CEF do
555
556
  insist { e.get('syslog') } == 'Syslogdate Sysloghost'
556
557
  end
557
558
  end
559
+
560
+ context "with raw_data_field set" do
561
+ subject(:codec) { LogStash::Codecs::CEF.new("raw_data_field" => "message_raw") }
562
+
563
+ it "should return the raw message in field message_raw" do
564
+ subject.decode(message) do |e|
565
+ validate(e)
566
+ insist { e.get("message_raw") } == message
567
+ end
568
+ end
569
+ end
570
+
558
571
  end
559
572
 
560
573
  context "decode with deprecated version option" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-cef
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.1.2
4
+ version: 4.1.3
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-26 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -44,7 +44,9 @@ dependencies:
44
44
  - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
- description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
47
+ description: This gem is a Logstash plugin required to be installed on top of the
48
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
49
+ gem is not a stand-alone program
48
50
  email: info@elastic.co
49
51
  executables: []
50
52
  extensions: []
@@ -56,6 +58,7 @@ files:
56
58
  - LICENSE
57
59
  - NOTICE.TXT
58
60
  - README.md
61
+ - docs/index.asciidoc
59
62
  - lib/logstash/codecs/cef.rb
60
63
  - logstash-codec-cef.gemspec
61
64
  - spec/codecs/cef_spec.rb
@@ -81,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
81
84
  version: '0'
82
85
  requirements: []
83
86
  rubyforge_project:
84
- rubygems_version: 2.4.8
87
+ rubygems_version: 2.6.11
85
88
  signing_key:
86
89
  specification_version: 4
87
90
  summary: CEF codec to parse and encode CEF formated logs