logstash-codec-idmef 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/docs/index.asciidoc +61 -22
  4. data/lib/logstash/codecs/idmef-message.dtd +661 -0
  5. data/lib/logstash/codecs/idmef.rb +283 -202
  6. data/logstash-codec-idmef.gemspec +1 -1
  7. data/spec/codecs/idmef_spec.rb +102 -12
  8. metadata +17 -17
data/lib/logstash/codecs/idmef.rb CHANGED
@@ -25,8 +25,8 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
25
25
  # tcp {
26
26
  # codec => idmef {
27
27
  # paths => {
28
- # "alert.classification.text" => "$message"
29
- # "alert.target(0).node.name" => "$host"
28
+ # "alert.classification.text" => "%{message}"
29
+ # "alert.target(0).node.name" => "%{host}"
30
30
  # "alert.analyzer(0).name" => "ACME"
31
31
  # }
32
32
  # }
@@ -37,33 +37,34 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
37
37
  # The keys of the hash are IDMEF path as described here:
38
38
  # https://redmine.secef.net/projects/secef/wiki/LibPrelude_IDMEF_path
39
39
  #
40
- # The values of the hash are values to set in final IDMEF. If a value starts with
41
- # a `$`, then the plugin try to retrieve the value from the event.
40
+ # The values of the hash are values to set in final IDMEF. If there is
41
+ # %{name} inside the string, the plugin try to retrieve the value from
42
+ # the event and create the final string.
42
43
  config :paths, :validate => :array, :default => {}
43
44
 
44
45
  # Try to use default paths mapping or not.
45
46
  #
46
47
  # Default paths are:
47
- # * alert.classification.text: ["$rule_name", "$event", "$message"]
48
- # * alert.detect_time: "$@timestamp"
49
- # * alert.create_time: "$@timestamp"
50
- # * alert.analyzer_time: "$@timestamp"
51
- # * alert.analyzer(0).name: ["$product", "$devname"]
52
- # * alert.analyzer(0).manufacturer: "$vendor"
53
- # * alert.source(0).node.address(0).address: ["$srcip", "$src"]
54
- # * alert.source(0).node.name: ["$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"]
55
- # * alert.source(0).service.port: ["$spt", "$sport", "$s_port"]
56
- # * alert.source(0).service.name: ["$sservice", "$srcservice"]
57
- # * alert.target(0).node.address(0).address: ["$hostip", "$dstip", "$dst", "$ip"]
58
- # * alert.target(0).node.name: ["$host", "$hostname", "$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"]
59
- # * alert.target(0).service.port: ["$dpt", "$dport", "$d_port"]
60
- # * alert.target(0).service.name: ["$service", "$service_id", "$dservice", "$dstservice",]
61
- # * alert.target(0).user.user_id(0).name: ["$user", "$dstuser", "$duser"]
62
- # * alert.target(0).user.user_id(0).number: ["$uid", "$dstuid", "$duid"]
63
- # * alert.target(0).process.name: ["$proc", "$process"]
64
- # * alert.target(0).process.pid: ["$dpid", "$pid"]
65
- # * alert.assessment.impact.severity: ["$severity", "$level"]
66
- # * alert.assessment.action.description: ["$action"]
48
+ # * "alert.analyzer(0).name": ["%{product}", "%{devname}"]
49
+ # * "alert.analyzer(0).manufacturer": ["%{vendor}"]
50
+ # * "alert.create_time": ["%{@timestamp}"]
51
+ # * "alert.detect_time": ["%{@timestamp}"]
52
+ # * "alert.analyzer_time": ["%{@timestamp}"]
53
+ # * "alert.source(0).node.address(0).address": ["%{srcip}", "%{src}"]
54
+ # * "alert.source(0).node.name": ["%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"]
55
+ # * "alert.source(0).service.port": ["%{spt}", "%{sport}", "%{s_port}"]
56
+ # * "alert.source(0).service.name": ["%{sservice}", "%{srcservice}"]
57
+ # * "alert.target(0).node.address(0).address": ["%{hostip}", "%{dstip}", "%{dst}", "%{ip}"]
58
+ # * "alert.target(0).node.name": ["%{host}", "%{hostname}", "%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"]
59
+ # * "alert.target(0).service.port": ["%{dpt}", "%{dport}", "%{d_port}"]
60
+ # * "alert.target(0).service.name": ["%{service}", "%{service_id}", "%{dservice}", "%{dstservice}"]
61
+ # * "alert.target(0).user.user_id(0).name": ["%{user}", "%{dstuser}", "%{duser}"]
62
+ # * "alert.target(0).user.user_id(0).number": ["%{uid}", "%{dstuid}", "%{duid}"]
63
+ # * "alert.target(0).process.name": ["%{proc}", "%{process}"]
64
+ # * "alert.target(0).process.pid": ["%{dpid}", "%{pid}"]
65
+ # * "alert.classification.text": ["%{rule_name}", "%{event}", "%{message}"]
66
+ # * "alert.assessment.impact.severity": ["%{severity}", "%{level}"]
67
+ # * "alert.assessment.action.description": ["%{action}"]
67
68
  config :defaults, :validate => :boolean, :default => true
68
69
 
69
70
  # When an alert is transformed in IDMEF, the remaining fields of the initial
@@ -71,6 +72,9 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
71
72
  # translation, set this setting to `false`.
72
73
  config :additionaldata, :validate => :boolean, :default => true
73
74
 
75
+ # Validate the generated XML with IDMEF DTD.
76
+ config :validate_xml, :validate => :boolean, :default => false
77
+
74
78
  # IDMEF can defined two types of message:
75
79
  # * alert
76
80
  #
@@ -91,8 +95,10 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
91
95
  # failed.
92
96
  config :type, :validate => :string, :default => "alert"
93
97
 
98
+ @@IDMEF_Time_Format = "%FT%T%:z"
99
+
94
100
  # RFC 4765: UserID Class
95
- IDMEFUserId = { :type => :class,
101
+ @@IDMEFUserId = { :type => :class,
96
102
  :name => "UserId",
97
103
  "name" => { :type => :list_value, :name => "name" },
98
104
  "type" => { :type => :attr, :name => "type", :default => "original-user" },
@@ -101,43 +107,43 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
101
107
  }
102
108
 
103
109
  # RFC 4765: User Class
104
- IDMEFUser = { :type => :class,
110
+ @@IDMEFUser = { :type => :class,
105
111
  :name => "User",
106
112
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
107
- "user_id" => { :type => :list_class, :class => IDMEFUserId }
113
+ "user_id" => { :type => :list_class, :class => @@IDMEFUserId }
108
114
  }
109
115
 
110
116
  # RFC 4765: FileAccess Class
111
- IDMEFFileAccess = { :type => :class,
117
+ @@IDMEFFileAccess = { :type => :class,
112
118
  :name => "FileAccess",
113
- "user_id" => { :type => :list_class, :class => IDMEFUserId }
119
+ "user_id" => { :type => :list_class, :class => @@IDMEFUserId }
114
120
  }
115
121
 
116
122
  # RFC 4765: File Class
117
- IDMEFFile = { :type => :class,
123
+ @@IDMEFFile = { :type => :class,
118
124
  :name => "File",
119
125
  "category" => { :type => :attr, :name => "category" },
120
126
  "fstype" => { :type => :attr, :name => "fstype" },
121
127
  "file-type" => { :type => :attr, :name => "file-type" },
122
128
  "name" => { :type => :list_value, :name => "name" },
123
129
  "path" => { :type => :list_value, :name => "path" },
124
- "file_access" => { :type => :list_class, :class => IDMEFFileAccess }
130
+ "file_access" => { :type => :list_class, :class => @@IDMEFFileAccess }
125
131
  }
126
132
 
127
133
  # RFC 4765: WebService Class
128
- IDMEFWebService = { :type => :class,
134
+ @@IDMEFWebService = { :type => :class,
129
135
  :name => "WebService",
130
136
  "url" => { :type => :list_value, :name => "url" }
131
137
  }
132
138
 
133
139
  # RFC 4765: SNMPService Class
134
- IDMEFSNMPService = { :type => :class,
140
+ @@IDMEFSNMPService = { :type => :class,
135
141
  :name => "SNMPService",
136
142
  "command" => { :type => :list_value, :name => "command" }
137
143
  }
138
144
 
139
145
  # RFC 4765: Service Class
140
- IDMEFService = { :type => :class,
146
+ @@IDMEFService = { :type => :class,
141
147
  :name => "Service",
142
148
  "ip_version" => { :type => :attr, :name => "ip_version" },
143
149
  "iana_protocol_number" => { :type => :attr, :name => "iana_protocol_number" },
@@ -146,12 +152,12 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
146
152
  "port" => { :type => :list_value, :name => "port" },
147
153
  "portlist" => { :type => :list_value, :name => "portlist" },
148
154
  "protocol" => { :type => :list_value, :name => "protocol" },
149
- "web_service" => { :type => :list_class, :class => IDMEFWebService },
150
- "snmp_service" => { :type => :list_class, :class => IDMEFSNMPService }
155
+ "web_service" => { :type => :list_class, :class => @@IDMEFWebService },
156
+ "snmp_service" => { :type => :list_class, :class => @@IDMEFSNMPService }
151
157
  }
152
158
 
153
159
  # RFC 4765: Address Class
154
- IDMEFAddress = { :type => :class,
160
+ @@IDMEFAddress = { :type => :class,
155
161
  :name => "Address",
156
162
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
157
163
  "vlan-name" => { :type => :attr, :name => "vlan-name" },
@@ -161,16 +167,16 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
161
167
  }
162
168
 
163
169
  # RFC 4765: Node Class
164
- IDMEFNode = { :type => :class,
170
+ @@IDMEFNode = { :type => :class,
165
171
  :name => "Node",
166
172
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
167
173
  "location" => { :type => :list_value, :name => "location" },
168
174
  "name" => { :type => :list_value, :name => "name" },
169
- "address" => { :type => :list_class, :class => IDMEFAddress },
175
+ "address" => { :type => :list_class, :class => @@IDMEFAddress },
170
176
  }
171
177
 
172
178
  # RFC 4765: Process Class
173
- IDMEFProcess = { :type => :class,
179
+ @@IDMEFProcess = { :type => :class,
174
180
  :name => "Process",
175
181
  "name" => { :type => :list_value, :name => "name" },
176
182
  "pid" => { :type => :list_value, :name => "pid" },
@@ -180,7 +186,7 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
180
186
  }
181
187
 
182
188
  # RFC 4765: Analyzer Class
183
- IDMEFAnalyzer = { :type => :class,
189
+ @@IDMEFAnalyzer = { :type => :class,
184
190
  :name => "Analyzer",
185
191
  "analyzerid" => { :type => :attr, :name => "analyzerid" },
186
192
  "name" => { :type => :attr, :name => "name" },
@@ -190,36 +196,36 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
190
196
  "class" => { :type => :attr, :name => "class" },
191
197
  "ostype" => { :type => :attr, :name => "ostype" },
192
198
  "osversion" => { :type => :attr, :name => "osversion" },
193
- "node" => { :type => :list_class, :class => IDMEFNode },
194
- "process" => { :type => :list_class, :class => IDMEFProcess },
199
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
200
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
195
201
  }
196
- IDMEFAnalyzer["analyzer"] = { :type => :list_class, :class => IDMEFAnalyzer }
202
+ @@IDMEFAnalyzer["analyzer"] = { :type => :list_class, :class => @@IDMEFAnalyzer }
197
203
 
198
204
  # RFC 4765: Source Class
199
- IDMEFSource = { :type => :class,
205
+ @@IDMEFSource = { :type => :class,
200
206
  :name => "Source",
201
207
  "spoofed" => { :type => :attr, :name => "spoofed", :default => "unknown" },
202
208
  "interface" => { :type => :attr, :name => "interface" },
203
- "node" => { :type => :list_class, :class => IDMEFNode },
204
- "user" => { :type => :list_class, :class => IDMEFUser },
205
- "process" => { :type => :list_class, :class => IDMEFProcess },
206
- "service" => { :type => :list_class, :class => IDMEFService },
209
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
210
+ "user" => { :type => :list_class, :class => @@IDMEFUser },
211
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
212
+ "service" => { :type => :list_class, :class => @@IDMEFService },
207
213
  }
208
214
 
209
215
  # RFC 4765: Target Class
210
- IDMEFTarget = { :type => :class,
216
+ @@IDMEFTarget = { :type => :class,
211
217
  :name => "Target",
212
218
  "decoy" => { :type => :attr, :name => "decoy", :default => "unknown" },
213
219
  "interface" => { :type => :attr, :name => "interface" },
214
- "node" => { :type => :list_class, :class => IDMEFNode },
215
- "user" => { :type => :list_class, :class => IDMEFUser },
216
- "process" => { :type => :list_class, :class => IDMEFProcess },
217
- "service" => { :type => :list_class, :class => IDMEFService },
218
- "file" => { :type => :list_class, :class => IDMEFFile }
220
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
221
+ "user" => { :type => :list_class, :class => @@IDMEFUser },
222
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
223
+ "service" => { :type => :list_class, :class => @@IDMEFService },
224
+ "file" => { :type => :list_class, :class => @@IDMEFFile }
219
225
  }
220
226
 
221
227
  # RFC 4765: Impact Class
222
- IDMEFImpact = { :type => :class,
228
+ @@IDMEFImpact = { :type => :class,
223
229
  :name => "Impact",
224
230
  "severity" => { :type => :attr, :name => "severity" },
225
231
  "completion" => { :type => :attr, :name => "completion" },
@@ -227,21 +233,21 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
227
233
  }
228
234
 
229
235
  # RFC 4765: Action Class
230
- IDMEFAction = { :type => :class,
236
+ @@IDMEFAction = { :type => :class,
231
237
  :name => "Action",
232
238
  "category" => { :type => :attr, :name => "category", :default => "other" },
233
239
  "description" => { :type => :value },
234
240
  }
235
241
 
236
242
  # RFC 4765: Confidence Class
237
- IDMEFConfidence = { :type => :class,
243
+ @@IDMEFConfidence = { :type => :class,
238
244
  :name => "Confidence",
239
245
  "rating" => { :type => :attr, :name => "rating", :default => "numeric" },
240
246
  "confidence" => { :type => :value },
241
247
  }
242
248
 
243
249
  # RFC 4765: Reference Class
244
- IDMEFReference = { :type => :class,
250
+ @@IDMEFReference = { :type => :class,
245
251
  :name => "Reference",
246
252
  "origin" => { :type => :attr, :name => "origin", :default => "unknown" },
247
253
  "meaning" => { :type => :attr, :name => "meaning" },
@@ -250,160 +256,217 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
250
256
  }
251
257
 
252
258
  # RFC 4765: AdditionalData Class
253
- IDMEFAdditionalData = { :type => :class,
259
+ @@IDMEFAdditionalData = { :type => :class,
254
260
  :name => "AdditionalData",
255
261
  "meaning" => { :type => :attr, :name => "meaning" },
256
262
  "type" => { :type => :attr, :name => "type" },
257
263
  "data" => { :type => :list_value, :name => :type }
258
264
  }
259
265
  # RFC 4765: CorrelationAlert Class
260
- IDMEFCorrelationAlert = { :type => :class,
266
+ @@IDMEFCorrelationAlert = { :type => :class,
261
267
  :name => "CorrelationAlert",
262
268
  "name" => { :type => :list_value, :name => "name" },
263
269
  "alertident" => { :type => :list_value, :name => "alertident" }
264
270
  }
265
271
 
266
272
  # RFC 4765: Assessment Class
267
- IDMEFAssessment = { :type => :class,
273
+ @@IDMEFAssessment = { :type => :class,
268
274
  :name => "Assessment",
269
- "impact" => { :type => :list_class, :class => IDMEFImpact },
270
- "action" => { :type => :list_class, :class => IDMEFAction },
271
- "confidence" => { :type => :list_class, :class => IDMEFConfidence }
275
+ "impact" => { :type => :list_class, :class => @@IDMEFImpact },
276
+ "action" => { :type => :list_class, :class => @@IDMEFAction },
277
+ "confidence" => { :type => :list_class, :class => @@IDMEFConfidence }
272
278
  }
273
279
 
274
280
  # RFC 4765: Classification Class
275
- IDMEFClassification = { :type => :class,
281
+ @@IDMEFClassification = { :type => :class,
276
282
  :name => "Classification",
277
283
  "text" => { :type => :attr, :name => "text" },
278
- "reference" => { :type => :list_class, :class => IDMEFReference }
284
+ "reference" => { :type => :list_class, :class => @@IDMEFReference }
279
285
  }
280
286
 
281
287
  # RFC 4765: Alert Class
282
- IDMEFAlert = { :type => :class,
288
+ @@IDMEFAlert = { :type => :class,
283
289
  :name => "Alert",
284
290
  "messageid" => { :type => :attr, :name => "messageid" },
285
291
  "create_time" => { :type => :list_value, :name => "CreateTime", :format => :datetime},
286
292
  "detect_time" => { :type => :list_value, :name => "DetectTime", :format => :datetime },
287
293
  "analyzer_time" => { :type => :list_value, :name => "AnalyzerTime", :format => :datetime },
288
- "analyzer" => { :type => :list_class, :class => IDMEFAnalyzer },
289
- "classification" => { :type => :list_class, :class => IDMEFClassification },
290
- "source" => { :type => :list_class, :class => IDMEFSource },
291
- "target" => { :type => :list_class, :class => IDMEFTarget },
292
- "assessment" => { :type => :list_class, :class => IDMEFAssessment },
293
- "additional_data" => { :type => :list_class, :class => IDMEFAdditionalData },
294
- "correlation_alert" => { :type => :list_class, :class => IDMEFCorrelationAlert },
294
+ "analyzer" => { :type => :list_class, :class => @@IDMEFAnalyzer },
295
+ "classification" => { :type => :list_class, :class => @@IDMEFClassification },
296
+ "source" => { :type => :list_class, :class => @@IDMEFSource },
297
+ "target" => { :type => :list_class, :class => @@IDMEFTarget },
298
+ "assessment" => { :type => :list_class, :class => @@IDMEFAssessment },
299
+ "additional_data" => { :type => :list_class, :class => @@IDMEFAdditionalData },
300
+ "correlation_alert" => { :type => :list_class, :class => @@IDMEFCorrelationAlert },
295
301
  }
296
302
 
297
303
  # RFC 4765: Message Class
298
- IDMEFMessage = { :type => :class,
304
+ @@IDMEFMessage = { :type => :class,
299
305
  :name => "IDMEF-Message",
300
- "alert" => { :type => :list_class, :class => IDMEFAlert },
306
+ "alert" => { :type => :list_class, :class => @@IDMEFAlert },
301
307
  }
308
+
309
+ @@local_paths = {
310
+ "alert.analyzer(0).name" => ["%{product}", "%{devname}"],
311
+ "alert.analyzer(0).manufacturer" => ["%{vendor}"],
312
+ "alert.create_time" => ["%{@timestamp}"],
313
+ "alert.detect_time" => ["%{@timestamp}"],
314
+ "alert.analyzer_time" => ["%{@timestamp}"],
315
+ "alert.source(0).node.address(0).address" => ["%{srcip}", "%{src}"],
316
+ "alert.source(0).node.name" => ["%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"],
317
+ "alert.source(0).service.port" => ["%{spt}", "%{sport}", "%{s_port}"],
318
+ "alert.source(0).service.name" => ["%{sservice}", "%{srcservice}"],
319
+ "alert.target(0).node.address(0).address" => ["%{hostip}", "%{dstip}", "%{dst}", "%{ip}"],
320
+ "alert.target(0).node.name" => ["%{host}", "%{hostname}", "%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"],
321
+ "alert.target(0).service.port" => ["%{dpt}", "%{dport}", "%{d_port}"],
322
+ "alert.target(0).service.name" => ["%{service}", "%{service_id}", "%{dservice}", "%{dstservice}"],
323
+ "alert.target(0).user.user_id(0).name" => ["%{user}", "%{dstuser}", "%{duser}"],
324
+ "alert.target(0).user.user_id(0).number" => ["%{uid}", "%{dstuid}", "%{duid}"],
325
+ "alert.target(0).process.name" => ["%{proc}", "%{process}"],
326
+ "alert.target(0).process.pid" => ["%{dpid}", "%{pid}"],
327
+ "alert.classification.text" => ["%{rule_name}", "%{event}", "%{message}"],
328
+ "alert.assessment.impact.severity" => ["%{severity}", "%{level}"],
329
+ "alert.assessment.action.description" => ["%{action}"],
330
+ }
331
+
302
332
  private
303
333
  def idmefpaths_to_xml(event, paths, doc = nil)
334
+ # create the document if not existing
304
335
  if doc.nil?
305
336
  doc = Nokogiri::XML::Document.new
337
+ if @validate_xml
338
+ doc.create_external_subset('IDMEF-Message', nil, @dtd_path)
339
+ end
306
340
  doc.root = Nokogiri::XML::Node.new('IDMEF-Message', doc)
307
341
  doc.root.add_namespace_definition('idmef', 'http://iana.org/idmef')
308
342
  end
309
- event_to_remove = []
310
- paths.each do |path, value|
311
- if !value.kind_of?(Array)
312
- value = [value]
313
- end
314
- value.each do |v|
315
- if v.to_s.start_with?("$")
316
- c = ''
317
- f = true
318
- v[1..-1].split('.').each do |ppath|
319
- if !event.get(c + '[' + ppath + ']').nil?
320
- c = c + '[' + ppath + ']'
321
- else
322
- f = false
323
- end
324
- end
325
- if !f then next end
326
- value = event.get(c)
327
- event_to_remove << c
328
- else
329
- value = v
330
- end
331
- end
332
- if value.kind_of?(Array) or value.to_s.empty?
333
- next
343
+
344
+ # translate all path inot the xml
345
+ paths.each do |path, values|
346
+ if !values.kind_of?(Array)
347
+ values = [values]
334
348
  end
335
- if value.kind_of?(String)
336
- @utf8_charset.convert(value)
349
+
350
+ formated_value = nil
351
+ values.each do |value|
352
+ formated_value = event.sprintf(value)
353
+ # value is looking for non existing variable in event
354
+ if /%{[^}]+}/.match(formated_value).nil?
355
+ break
356
+ end
357
+
358
+ if formated_value == value
359
+ formated_value = nil
360
+ end
337
361
  end
338
- curr = doc.root
339
- rfc = IDMEFMessage
340
- path.split('.').each do |name|
341
- ret = name.match(/^(.*)\((\d+)\)/)
342
- if ret
343
- name = ret[1]
344
- v = (ret ? ret[2] : 0).to_i
362
+
363
+ next if formated_value.nil? or formated_value.empty?
364
+
365
+ @utf8_charset.convert(formated_value)
366
+
367
+ xml_current_node = doc.root
368
+ rfc_current_class = @@IDMEFMessage
369
+ # path is an idmef path. example : alert.classification.text
370
+ path.split('.').each do |idmefpath_name|
371
+ # handle listed_path like alert.target(0).node.address(0).address
372
+ listed_path = idmefpath_name.match(/^(.*)\((\d+)\)/)
373
+ idmefpath_index = nil
374
+ if listed_path
375
+ idmefpath_name = listed_path[1]
376
+ idmefpath_index = (listed_path ? listed_path[2] : 0).to_i
345
377
  end
346
378
 
347
- ne = rfc[name][:class]
348
- ne_t = rfc[name][:type]
349
- path_idmef = rfc[name][:type] == :list_class ? ne[:name] : nil
379
+ idmefpath_rfc_elm = rfc_current_class[idmefpath_name]
380
+
381
+ idmef_node_name = nil
382
+ if rfc_current_class[idmefpath_name][:type] == :list_class
383
+ idmef_node_name = idmefpath_rfc_elm[:class][:name]
384
+ end
350
385
 
351
- if ret && ne_t == :list_class
352
- c = curr.xpath(path_idmef)
353
- if c.empty?
354
- no = Nokogiri::XML::Node.new(ne[:name], doc)
355
- curr << no
356
- elsif c.length <= v
357
- nod = c[-1]
358
- (c.length..v).each do |t|
359
- no = Nokogiri::XML::Node.new(ne[:name], doc)
360
- nod.after(no)
361
- nod = no
386
+ # rfc class with multiple elements
387
+ if !idmefpath_index.nil? && idmefpath_rfc_elm[:type] == :list_class
388
+ idmef_nodes = xml_current_node.xpath(idmef_node_name)
389
+
390
+ if idmef_nodes.empty?
391
+ idmef_node = Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
392
+ xml_current_node << idmef_node
393
+
394
+ elsif idmef_nodes.length <= idmefpath_index
395
+ idmef_node = idmef_nodes[-1]
396
+ (idmef_nodes.length..idmefpath_index).each do |idx|
397
+ tmp_node = Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
398
+ idmef_node.after(tmp_node)
399
+ idmef_node = tmp_node
362
400
  end
363
- elsif c.length > v
364
- no = c[v]
401
+
402
+ elsif idmef_nodes.length > idmefpath_index
403
+ idmef_node = idmef_nodes[idmefpath_index]
365
404
  end
366
- curr = no
367
- rfc = ne
368
- elsif !ret && ne_t == :list_class
369
- no = curr.xpath(path_idmef).first || Nokogiri::XML::Node.new(ne[:name], doc)
370
- curr << no
371
- curr = no
372
- rfc = ne
373
- elsif ne_t == :list_value
374
- if rfc[name][:format] == :datetime
375
- value = DateTime.parse(value.to_s).strftime("%FT%T%:z")
405
+
406
+ xml_current_node = idmef_node
407
+ rfc_current_class = idmefpath_rfc_elm[:class]
408
+
409
+ # rfc class with on element
410
+ elsif idmefpath_index.nil? && idmefpath_rfc_elm[:type] == :list_class
411
+ idmef_node = xml_current_node.xpath(idmef_node_name).first
412
+ idmef_node = idmef_node || Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
413
+ xml_current_node << idmef_node
414
+ xml_current_node = idmef_node
415
+ rfc_current_class = idmefpath_rfc_elm[:class]
416
+
417
+ # rfc multiple values
418
+ elsif idmefpath_rfc_elm[:type] == :list_value
419
+ if rfc_current_class[idmefpath_name][:name] == :type
420
+ node_name = xml_current_node["type"]
421
+ else
422
+ node_name = rfc_current_class[idmefpath_name][:name]
376
423
  end
377
- n = rfc[name][:name] == :type ? curr["type"] : rfc[name][:name]
378
- no = Nokogiri::XML::Node.new(n, doc)
379
- no.content = value.to_s
380
- curr << no
381
- elsif ne_t == :attr
382
- if rfc[name][:format] == :datetime
383
- value = DateTime.parse(value.to_s).strftime("%FT%T%:z")
424
+
425
+ idmef_node = Nokogiri::XML::Node.new(node_name, doc)
426
+
427
+ # reformat datetime with the expected format described in idmef rfc
428
+ if rfc_current_class[idmefpath_name][:format] == :datetime
429
+ alert_time = DateTime.parse(formated_value)
430
+ formated_value = alert_time.strftime(@@IDMEF_Time_Format)
431
+ seconds = alert_time.to_time.to_i + 2208988800
432
+ seconds_fraction = (alert_time.to_time.usec. / (1000000.0 / (2 ** 32))).to_i
433
+ idmef_node["ntpstamp"] = "0x%08x.0x%08x" % [seconds, seconds_fraction]
384
434
  end
385
- curr[rfc[name][:name]] = value.to_s
435
+
436
+ idmef_node.content = formated_value
437
+
438
+ xml_current_node << idmef_node
439
+
440
+ # rfc attribute
441
+ elsif idmefpath_rfc_elm[:type] == :attr
442
+ xml_current_node[rfc_current_class[idmefpath_name][:name]] = formated_value.to_s
443
+
386
444
  end
387
- rfc.each do |kk, vv|
388
- if vv.respond_to?(:each_pair) && vv[:default] && vv[:type] == :attr && !curr[vv[:name]]
389
- curr[vv[:name]] = vv[:default]
445
+
446
+ # set default values as described in rfc
447
+ rfc_current_class.each do |element, value|
448
+ # value is a ref, a string or a hash, we want hashs
449
+ next if !value.respond_to?(:each_pair)
450
+
451
+ if value[:default] && value[:type] == :attr && !xml_current_node[value[:name]]
452
+ xml_current_node[value[:name]] = value[:default]
390
453
  end
391
454
  end
392
455
  end
393
456
  end
394
- event_to_remove.each do |v|
395
- event.remove(v)
396
- end
397
457
  return doc
398
458
  end
399
459
 
400
460
  private
401
461
  def xml_to_string(doc)
402
- doc.root.traverse { |node|
462
+ # add namespace "idmef"
463
+ doc.root.traverse do |node|
403
464
  if node.type != Nokogiri::XML::Node::TEXT_NODE
404
465
  node.name = 'idmef:' + node.name
405
466
  end
406
- }
467
+ end
468
+
469
+ # return a oneline xml without spaces
407
470
  return doc.serialize(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML).sub("\n", "").strip
408
471
  end
409
472
 
@@ -413,75 +476,93 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
413
476
  @utf8_charset = LogStash::Util::Charset.new('UTF-8')
414
477
  @utf8_charset.logger = self.logger
415
478
 
416
- @local_paths = {
417
- "alert.analyzer(0).name" => ["$product", "$devname"],
418
- "alert.analyzer(0).manufacturer" => "$vendor",
419
- "alert.create_time" => "$@timestamp",
420
- "alert.detect_time" => "$@timestamp",
421
- "alert.analyzer_time" => "$@timestamp",
422
- "alert.source(0).node.address(0).address" => ["$srcip", "$src"],
423
- "alert.source(0).node.name" => ["$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"],
424
- "alert.source(0).service.port" => ["$spt", "$sport", "$s_port"],
425
- "alert.source(0).service.name" => ["$sservice", "$srcservice"],
426
- "alert.target(0).node.address(0).address" => ["$hostip", "$dstip", "$dst", "$ip"],
427
- "alert.target(0).node.name" => ["$host", "$hostname", "$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"],
428
- "alert.target(0).service.port" => ["$dpt", "$dport", "$d_port"],
429
- "alert.target(0).service.name" => ["$service", "$service_id", "$dservice", "$dstservice",],
430
- "alert.target(0).user.user_id(0).name" => ["$user", "$dstuser", "$duser"],
431
- "alert.target(0).user.user_id(0).number" => ["$uid", "$dstuid", "$duid"],
432
- "alert.target(0).process.name" => ["$proc", "$process"],
433
- "alert.target(0).process.pid" => ["$dpid", "$pid"],
434
- "alert.classification.text" => ["$rule_name", "$event", "$message"],
435
- "alert.assessment.impact.severity" => ["$severity", "$level"],
436
- "alert.assessment.action.description" => ["$action"],
437
- }
438
479
  if @defaults
439
- @allpaths = @local_paths.merge(@paths)
480
+ @allpaths = @@local_paths.merge(@paths)
440
481
  else
441
482
  @allpaths = @paths
442
483
  end
484
+
485
+ if @additionaldata
486
+ # Find all event's keys already used in @@IDMEF paths values
487
+ @allpaths_event_keys = []
488
+ @allpaths.each do |key, values|
489
+ if !values.kind_of?(Array)
490
+ values = [values]
491
+ end
492
+
493
+ values.each do |value|
494
+ match = value.match(/%{([^}]+)}/)
495
+ if match
496
+ @allpaths_event_keys += match.captures
497
+ end
498
+ end
499
+ end
500
+ end
501
+
502
+ if @validate_xml
503
+ @dtd_path = File.dirname(File.expand_path(__FILE__)) + "/idmef-message.dtd"
504
+ @dtd_options = Nokogiri::XML::ParseOptions.new()
505
+ @dtd_options.recover
506
+ @dtd_options.dtdload
507
+ @dtd_options.dtdvalid
508
+ end
443
509
  end
444
510
 
445
511
  public
446
512
  def encode(event)
447
- # Reload configuration
448
- @allpaths = @allpaths.merge(@paths)
513
+ # Set messageid and analyzerid
514
+ paths = { "%s.messageid" % @type => java.util.UUID.randomUUID.to_s,
515
+ "%s.analyzer(0).analyzerid" % @type => Socket.gethostname.to_s
516
+ }
449
517
 
450
- # Copy event
451
- e = event.clone
518
+ # CreateTime is required in IDMEF RFC
519
+ if !@allpaths.include? "alert.create_time"
520
+ paths["alert.create_time"] = DateTime.now().strftime(@@IDMEF_Time_Format)
521
+ end
452
522
 
453
- # Set messageid and analyzerid
454
- p = { "%s.messageid" % @type => java.util.UUID.randomUUID.to_s,
455
- "%s.analyzer(0).analyzerid" % @type => Socket.gethostname.to_s
456
- }
457
- xml = idmefpaths_to_xml(e, p)
523
+ # Classification is required in IDMEF RFC
524
+ if !@allpaths.include? "alert.classification.text"
525
+ paths["alert.classification.text"] = "Unknown alert"
526
+ end
527
+
528
+ xml = idmefpaths_to_xml(event, paths)
458
529
 
459
- # Set paths
460
- xml = idmefpaths_to_xml(e, @allpaths, xml)
530
+ # Set configured paths
531
+ xml = idmefpaths_to_xml(event, @allpaths, xml)
461
532
 
462
- # Set Additional data
533
+ # Add unused event data to IDMEF additional data
463
534
  if @additionaldata
464
- idx = xml.xpath('/IDMEF-Message/Alert/AddionnalData').length
465
- e.to_hash.each do |key, value|
466
- if value.to_s.empty?
467
- next
468
- end
535
+ additionaldata_idx = xml.xpath('/idmef-message/alert/addionnaldata').length
536
+
537
+ event.to_hash.each do |key, value|
538
+ next if value.to_s.empty? or @allpaths_event_keys.include? key
539
+
469
540
  if value.kind_of?(Integer)
470
- t = "integer"
541
+ value_type = "integer"
471
542
  elsif value.kind_of?(Float)
472
- t = "real"
543
+ value_type = "real"
473
544
  else
474
- t = "string"
545
+ value_type = "string"
475
546
  end
476
- p = { "alert.additional_data(%d).meaning" % idx => key,
477
- "alert.additional_data(%d).type" % idx => t,
478
- "alert.additional_data(%d).data" % idx => value.to_s,
479
- }
480
- xml = idmefpaths_to_xml(e, p , xml)
481
- idx = idx + 1
547
+
548
+ paths = { "alert.additional_data(%d).meaning" % additionaldata_idx => key,
549
+ "alert.additional_data(%d).type" % additionaldata_idx => value_type,
550
+ "alert.additional_data(%d).data" % additionaldata_idx => value.to_s,
551
+ }
552
+
553
+ xml = idmefpaths_to_xml(event, paths , xml)
554
+ additionaldata_idx += 1
482
555
  end
483
556
  end
484
557
 
558
+ if @validate_xml
559
+ xml_dtd = Nokogiri::XML.parse(xml.to_xml, nil, nil, @dtd_options)
560
+ if !xml_dtd.validate.nil? and !xml_dtd.validate.empty?
561
+ raise "IDMEF XML generated is not valid. Errors: %s." % xml_dtd.validate.join(', ')
562
+ end
563
+ xml.external_subset.remove
564
+ end
565
+
485
566
  # Create the XML
486
567
  @on_event.call(event, xml_to_string(xml) + NL)
487
568
  end