logstash-codec-idmef 0.9.2 → 0.9.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (8) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/docs/index.asciidoc +61 -22
  4. data/lib/logstash/codecs/idmef-message.dtd +661 -0
  5. data/lib/logstash/codecs/idmef.rb +283 -202
  6. data/logstash-codec-idmef.gemspec +1 -1
  7. data/spec/codecs/idmef_spec.rb +102 -12
  8. metadata +17 -17
data/lib/logstash/codecs/idmef.rb CHANGED
@@ -25,8 +25,8 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
25
25
  # tcp {
26
26
  # codec => idmef {
27
27
  # paths => {
28
- # "alert.classification.text" => "$message"
29
- # "alert.target(0).node.name" => "$host"
28
+ # "alert.classification.text" => "%{message}"
29
+ # "alert.target(0).node.name" => "%{host}"
30
30
  # "alert.analyzer(0).name" => "ACME"
31
31
  # }
32
32
  # }
@@ -37,33 +37,34 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
37
37
  # The keys of the hash are IDMEF path as described here:
38
38
  # https://redmine.secef.net/projects/secef/wiki/LibPrelude_IDMEF_path
39
39
  #
40
- # The values of the hash are values to set in final IDMEF. If a value starts with
41
- # a `$`, then the plugin try to retrieve the value from the event.
40
+ # The values of the hash are values to set in final IDMEF. If there is
41
+ # %{name} inside the string, the plugin try to retrieve the value from
42
+ # the event and create the final string.
42
43
  config :paths, :validate => :array, :default => {}
43
44
 
44
45
  # Try to use default paths mapping or not.
45
46
  #
46
47
  # Default paths are:
47
- # * alert.classification.text: ["$rule_name", "$event", "$message"]
48
- # * alert.detect_time: "$@timestamp"
49
- # * alert.create_time: "$@timestamp"
50
- # * alert.analyzer_time: "$@timestamp"
51
- # * alert.analyzer(0).name: ["$product", "$devname"]
52
- # * alert.analyzer(0).manufacturer: "$vendor"
53
- # * alert.source(0).node.address(0).address: ["$srcip", "$src"]
54
- # * alert.source(0).node.name: ["$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"]
55
- # * alert.source(0).service.port: ["$spt", "$sport", "$s_port"]
56
- # * alert.source(0).service.name: ["$sservice", "$srcservice"]
57
- # * alert.target(0).node.address(0).address: ["$hostip", "$dstip", "$dst", "$ip"]
58
- # * alert.target(0).node.name: ["$host", "$hostname", "$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"]
59
- # * alert.target(0).service.port: ["$dpt", "$dport", "$d_port"]
60
- # * alert.target(0).service.name: ["$service", "$service_id", "$dservice", "$dstservice",]
61
- # * alert.target(0).user.user_id(0).name: ["$user", "$dstuser", "$duser"]
62
- # * alert.target(0).user.user_id(0).number: ["$uid", "$dstuid", "$duid"]
63
- # * alert.target(0).process.name: ["$proc", "$process"]
64
- # * alert.target(0).process.pid: ["$dpid", "$pid"]
65
- # * alert.assessment.impact.severity: ["$severity", "$level"]
66
- # * alert.assessment.action.description: ["$action"]
48
+ # * "alert.analyzer(0).name": ["%{product}", "%{devname}"]
49
+ # * "alert.analyzer(0).manufacturer": ["%{vendor}"]
50
+ # * "alert.create_time": ["%{@timestamp}"]
51
+ # * "alert.detect_time": ["%{@timestamp}"]
52
+ # * "alert.analyzer_time": ["%{@timestamp}"]
53
+ # * "alert.source(0).node.address(0).address": ["%{srcip}", "%{src}"]
54
+ # * "alert.source(0).node.name": ["%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"]
55
+ # * "alert.source(0).service.port": ["%{spt}", "%{sport}", "%{s_port}"]
56
+ # * "alert.source(0).service.name": ["%{sservice}", "%{srcservice}"]
57
+ # * "alert.target(0).node.address(0).address": ["%{hostip}", "%{dstip}", "%{dst}", "%{ip}"]
58
+ # * "alert.target(0).node.name": ["%{host}", "%{hostname}", "%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"]
59
+ # * "alert.target(0).service.port": ["%{dpt}", "%{dport}", "%{d_port}"]
60
+ # * "alert.target(0).service.name": ["%{service}", "%{service_id}", "%{dservice}", "%{dstservice}"]
61
+ # * "alert.target(0).user.user_id(0).name": ["%{user}", "%{dstuser}", "%{duser}"]
62
+ # * "alert.target(0).user.user_id(0).number": ["%{uid}", "%{dstuid}", "%{duid}"]
63
+ # * "alert.target(0).process.name": ["%{proc}", "%{process}"]
64
+ # * "alert.target(0).process.pid": ["%{dpid}", "%{pid}"]
65
+ # * "alert.classification.text": ["%{rule_name}", "%{event}", "%{message}"]
66
+ # * "alert.assessment.impact.severity": ["%{severity}", "%{level}"]
67
+ # * "alert.assessment.action.description": ["%{action}"]
67
68
  config :defaults, :validate => :boolean, :default => true
68
69
 
69
70
  # When an alert is transformed in IDMEF, the remaining fields of the initial
@@ -71,6 +72,9 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
71
72
  # translation, set this setting to `false`.
72
73
  config :additionaldata, :validate => :boolean, :default => true
73
74
 
75
+ # Validate the generated XML with IDMEF DTD.
76
+ config :validate_xml, :validate => :boolean, :default => false
77
+
74
78
  # IDMEF can defined two types of message:
75
79
  # * alert
76
80
  #
@@ -91,8 +95,10 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
91
95
  # failed.
92
96
  config :type, :validate => :string, :default => "alert"
93
97
 
98
+ @@IDMEF_Time_Format = "%FT%T%:z"
99
+
94
100
  # RFC 4765: UserID Class
95
- IDMEFUserId = { :type => :class,
101
+ @@IDMEFUserId = { :type => :class,
96
102
  :name => "UserId",
97
103
  "name" => { :type => :list_value, :name => "name" },
98
104
  "type" => { :type => :attr, :name => "type", :default => "original-user" },
@@ -101,43 +107,43 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
101
107
  }
102
108
 
103
109
  # RFC 4765: User Class
104
- IDMEFUser = { :type => :class,
110
+ @@IDMEFUser = { :type => :class,
105
111
  :name => "User",
106
112
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
107
- "user_id" => { :type => :list_class, :class => IDMEFUserId }
113
+ "user_id" => { :type => :list_class, :class => @@IDMEFUserId }
108
114
  }
109
115
 
110
116
  # RFC 4765: FileAccess Class
111
- IDMEFFileAccess = { :type => :class,
117
+ @@IDMEFFileAccess = { :type => :class,
112
118
  :name => "FileAccess",
113
- "user_id" => { :type => :list_class, :class => IDMEFUserId }
119
+ "user_id" => { :type => :list_class, :class => @@IDMEFUserId }
114
120
  }
115
121
 
116
122
  # RFC 4765: File Class
117
- IDMEFFile = { :type => :class,
123
+ @@IDMEFFile = { :type => :class,
118
124
  :name => "File",
119
125
  "category" => { :type => :attr, :name => "category" },
120
126
  "fstype" => { :type => :attr, :name => "fstype" },
121
127
  "file-type" => { :type => :attr, :name => "file-type" },
122
128
  "name" => { :type => :list_value, :name => "name" },
123
129
  "path" => { :type => :list_value, :name => "path" },
124
- "file_access" => { :type => :list_class, :class => IDMEFFileAccess }
130
+ "file_access" => { :type => :list_class, :class => @@IDMEFFileAccess }
125
131
  }
126
132
 
127
133
  # RFC 4765: WebService Class
128
- IDMEFWebService = { :type => :class,
134
+ @@IDMEFWebService = { :type => :class,
129
135
  :name => "WebService",
130
136
  "url" => { :type => :list_value, :name => "url" }
131
137
  }
132
138
 
133
139
  # RFC 4765: SNMPService Class
134
- IDMEFSNMPService = { :type => :class,
140
+ @@IDMEFSNMPService = { :type => :class,
135
141
  :name => "SNMPService",
136
142
  "command" => { :type => :list_value, :name => "command" }
137
143
  }
138
144
 
139
145
  # RFC 4765: Service Class
140
- IDMEFService = { :type => :class,
146
+ @@IDMEFService = { :type => :class,
141
147
  :name => "Service",
142
148
  "ip_version" => { :type => :attr, :name => "ip_version" },
143
149
  "iana_protocol_number" => { :type => :attr, :name => "iana_protocol_number" },
@@ -146,12 +152,12 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
146
152
  "port" => { :type => :list_value, :name => "port" },
147
153
  "portlist" => { :type => :list_value, :name => "portlist" },
148
154
  "protocol" => { :type => :list_value, :name => "protocol" },
149
- "web_service" => { :type => :list_class, :class => IDMEFWebService },
150
- "snmp_service" => { :type => :list_class, :class => IDMEFSNMPService }
155
+ "web_service" => { :type => :list_class, :class => @@IDMEFWebService },
156
+ "snmp_service" => { :type => :list_class, :class => @@IDMEFSNMPService }
151
157
  }
152
158
 
153
159
  # RFC 4765: Address Class
154
- IDMEFAddress = { :type => :class,
160
+ @@IDMEFAddress = { :type => :class,
155
161
  :name => "Address",
156
162
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
157
163
  "vlan-name" => { :type => :attr, :name => "vlan-name" },
@@ -161,16 +167,16 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
161
167
  }
162
168
 
163
169
  # RFC 4765: Node Class
164
- IDMEFNode = { :type => :class,
170
+ @@IDMEFNode = { :type => :class,
165
171
  :name => "Node",
166
172
  "category" => { :type => :attr, :name => "category", :default => "unknown" },
167
173
  "location" => { :type => :list_value, :name => "location" },
168
174
  "name" => { :type => :list_value, :name => "name" },
169
- "address" => { :type => :list_class, :class => IDMEFAddress },
175
+ "address" => { :type => :list_class, :class => @@IDMEFAddress },
170
176
  }
171
177
 
172
178
  # RFC 4765: Process Class
173
- IDMEFProcess = { :type => :class,
179
+ @@IDMEFProcess = { :type => :class,
174
180
  :name => "Process",
175
181
  "name" => { :type => :list_value, :name => "name" },
176
182
  "pid" => { :type => :list_value, :name => "pid" },
@@ -180,7 +186,7 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
180
186
  }
181
187
 
182
188
  # RFC 4765: Analyzer Class
183
- IDMEFAnalyzer = { :type => :class,
189
+ @@IDMEFAnalyzer = { :type => :class,
184
190
  :name => "Analyzer",
185
191
  "analyzerid" => { :type => :attr, :name => "analyzerid" },
186
192
  "name" => { :type => :attr, :name => "name" },
@@ -190,36 +196,36 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
190
196
  "class" => { :type => :attr, :name => "class" },
191
197
  "ostype" => { :type => :attr, :name => "ostype" },
192
198
  "osversion" => { :type => :attr, :name => "osversion" },
193
- "node" => { :type => :list_class, :class => IDMEFNode },
194
- "process" => { :type => :list_class, :class => IDMEFProcess },
199
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
200
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
195
201
  }
196
- IDMEFAnalyzer["analyzer"] = { :type => :list_class, :class => IDMEFAnalyzer }
202
+ @@IDMEFAnalyzer["analyzer"] = { :type => :list_class, :class => @@IDMEFAnalyzer }
197
203
 
198
204
  # RFC 4765: Source Class
199
- IDMEFSource = { :type => :class,
205
+ @@IDMEFSource = { :type => :class,
200
206
  :name => "Source",
201
207
  "spoofed" => { :type => :attr, :name => "spoofed", :default => "unknown" },
202
208
  "interface" => { :type => :attr, :name => "interface" },
203
- "node" => { :type => :list_class, :class => IDMEFNode },
204
- "user" => { :type => :list_class, :class => IDMEFUser },
205
- "process" => { :type => :list_class, :class => IDMEFProcess },
206
- "service" => { :type => :list_class, :class => IDMEFService },
209
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
210
+ "user" => { :type => :list_class, :class => @@IDMEFUser },
211
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
212
+ "service" => { :type => :list_class, :class => @@IDMEFService },
207
213
  }
208
214
 
209
215
  # RFC 4765: Target Class
210
- IDMEFTarget = { :type => :class,
216
+ @@IDMEFTarget = { :type => :class,
211
217
  :name => "Target",
212
218
  "decoy" => { :type => :attr, :name => "decoy", :default => "unknown" },
213
219
  "interface" => { :type => :attr, :name => "interface" },
214
- "node" => { :type => :list_class, :class => IDMEFNode },
215
- "user" => { :type => :list_class, :class => IDMEFUser },
216
- "process" => { :type => :list_class, :class => IDMEFProcess },
217
- "service" => { :type => :list_class, :class => IDMEFService },
218
- "file" => { :type => :list_class, :class => IDMEFFile }
220
+ "node" => { :type => :list_class, :class => @@IDMEFNode },
221
+ "user" => { :type => :list_class, :class => @@IDMEFUser },
222
+ "process" => { :type => :list_class, :class => @@IDMEFProcess },
223
+ "service" => { :type => :list_class, :class => @@IDMEFService },
224
+ "file" => { :type => :list_class, :class => @@IDMEFFile }
219
225
  }
220
226
 
221
227
  # RFC 4765: Impact Class
222
- IDMEFImpact = { :type => :class,
228
+ @@IDMEFImpact = { :type => :class,
223
229
  :name => "Impact",
224
230
  "severity" => { :type => :attr, :name => "severity" },
225
231
  "completion" => { :type => :attr, :name => "completion" },
@@ -227,21 +233,21 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
227
233
  }
228
234
 
229
235
  # RFC 4765: Action Class
230
- IDMEFAction = { :type => :class,
236
+ @@IDMEFAction = { :type => :class,
231
237
  :name => "Action",
232
238
  "category" => { :type => :attr, :name => "category", :default => "other" },
233
239
  "description" => { :type => :value },
234
240
  }
235
241
 
236
242
  # RFC 4765: Confidence Class
237
- IDMEFConfidence = { :type => :class,
243
+ @@IDMEFConfidence = { :type => :class,
238
244
  :name => "Confidence",
239
245
  "rating" => { :type => :attr, :name => "rating", :default => "numeric" },
240
246
  "confidence" => { :type => :value },
241
247
  }
242
248
 
243
249
  # RFC 4765: Reference Class
244
- IDMEFReference = { :type => :class,
250
+ @@IDMEFReference = { :type => :class,
245
251
  :name => "Reference",
246
252
  "origin" => { :type => :attr, :name => "origin", :default => "unknown" },
247
253
  "meaning" => { :type => :attr, :name => "meaning" },
@@ -250,160 +256,217 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
250
256
  }
251
257
 
252
258
  # RFC 4765: AdditionalData Class
253
- IDMEFAdditionalData = { :type => :class,
259
+ @@IDMEFAdditionalData = { :type => :class,
254
260
  :name => "AdditionalData",
255
261
  "meaning" => { :type => :attr, :name => "meaning" },
256
262
  "type" => { :type => :attr, :name => "type" },
257
263
  "data" => { :type => :list_value, :name => :type }
258
264
  }
259
265
  # RFC 4765: CorrelationAlert Class
260
- IDMEFCorrelationAlert = { :type => :class,
266
+ @@IDMEFCorrelationAlert = { :type => :class,
261
267
  :name => "CorrelationAlert",
262
268
  "name" => { :type => :list_value, :name => "name" },
263
269
  "alertident" => { :type => :list_value, :name => "alertident" }
264
270
  }
265
271
 
266
272
  # RFC 4765: Assessment Class
267
- IDMEFAssessment = { :type => :class,
273
+ @@IDMEFAssessment = { :type => :class,
268
274
  :name => "Assessment",
269
- "impact" => { :type => :list_class, :class => IDMEFImpact },
270
- "action" => { :type => :list_class, :class => IDMEFAction },
271
- "confidence" => { :type => :list_class, :class => IDMEFConfidence }
275
+ "impact" => { :type => :list_class, :class => @@IDMEFImpact },
276
+ "action" => { :type => :list_class, :class => @@IDMEFAction },
277
+ "confidence" => { :type => :list_class, :class => @@IDMEFConfidence }
272
278
  }
273
279
 
274
280
  # RFC 4765: Classification Class
275
- IDMEFClassification = { :type => :class,
281
+ @@IDMEFClassification = { :type => :class,
276
282
  :name => "Classification",
277
283
  "text" => { :type => :attr, :name => "text" },
278
- "reference" => { :type => :list_class, :class => IDMEFReference }
284
+ "reference" => { :type => :list_class, :class => @@IDMEFReference }
279
285
  }
280
286
 
281
287
  # RFC 4765: Alert Class
282
- IDMEFAlert = { :type => :class,
288
+ @@IDMEFAlert = { :type => :class,
283
289
  :name => "Alert",
284
290
  "messageid" => { :type => :attr, :name => "messageid" },
285
291
  "create_time" => { :type => :list_value, :name => "CreateTime", :format => :datetime},
286
292
  "detect_time" => { :type => :list_value, :name => "DetectTime", :format => :datetime },
287
293
  "analyzer_time" => { :type => :list_value, :name => "AnalyzerTime", :format => :datetime },
288
- "analyzer" => { :type => :list_class, :class => IDMEFAnalyzer },
289
- "classification" => { :type => :list_class, :class => IDMEFClassification },
290
- "source" => { :type => :list_class, :class => IDMEFSource },
291
- "target" => { :type => :list_class, :class => IDMEFTarget },
292
- "assessment" => { :type => :list_class, :class => IDMEFAssessment },
293
- "additional_data" => { :type => :list_class, :class => IDMEFAdditionalData },
294
- "correlation_alert" => { :type => :list_class, :class => IDMEFCorrelationAlert },
294
+ "analyzer" => { :type => :list_class, :class => @@IDMEFAnalyzer },
295
+ "classification" => { :type => :list_class, :class => @@IDMEFClassification },
296
+ "source" => { :type => :list_class, :class => @@IDMEFSource },
297
+ "target" => { :type => :list_class, :class => @@IDMEFTarget },
298
+ "assessment" => { :type => :list_class, :class => @@IDMEFAssessment },
299
+ "additional_data" => { :type => :list_class, :class => @@IDMEFAdditionalData },
300
+ "correlation_alert" => { :type => :list_class, :class => @@IDMEFCorrelationAlert },
295
301
  }
296
302
 
297
303
  # RFC 4765: Message Class
298
- IDMEFMessage = { :type => :class,
304
+ @@IDMEFMessage = { :type => :class,
299
305
  :name => "IDMEF-Message",
300
- "alert" => { :type => :list_class, :class => IDMEFAlert },
306
+ "alert" => { :type => :list_class, :class => @@IDMEFAlert },
301
307
  }
308
+
309
+ @@local_paths = {
310
+ "alert.analyzer(0).name" => ["%{product}", "%{devname}"],
311
+ "alert.analyzer(0).manufacturer" => ["%{vendor}"],
312
+ "alert.create_time" => ["%{@timestamp}"],
313
+ "alert.detect_time" => ["%{@timestamp}"],
314
+ "alert.analyzer_time" => ["%{@timestamp}"],
315
+ "alert.source(0).node.address(0).address" => ["%{srcip}", "%{src}"],
316
+ "alert.source(0).node.name" => ["%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"],
317
+ "alert.source(0).service.port" => ["%{spt}", "%{sport}", "%{s_port}"],
318
+ "alert.source(0).service.name" => ["%{sservice}", "%{srcservice}"],
319
+ "alert.target(0).node.address(0).address" => ["%{hostip}", "%{dstip}", "%{dst}", "%{ip}"],
320
+ "alert.target(0).node.name" => ["%{host}", "%{hostname}", "%{shost}", "%{srchost}", "%{shostname}", "%{srchostname}", "%{sname}", "%{srcname}"],
321
+ "alert.target(0).service.port" => ["%{dpt}", "%{dport}", "%{d_port}"],
322
+ "alert.target(0).service.name" => ["%{service}", "%{service_id}", "%{dservice}", "%{dstservice}"],
323
+ "alert.target(0).user.user_id(0).name" => ["%{user}", "%{dstuser}", "%{duser}"],
324
+ "alert.target(0).user.user_id(0).number" => ["%{uid}", "%{dstuid}", "%{duid}"],
325
+ "alert.target(0).process.name" => ["%{proc}", "%{process}"],
326
+ "alert.target(0).process.pid" => ["%{dpid}", "%{pid}"],
327
+ "alert.classification.text" => ["%{rule_name}", "%{event}", "%{message}"],
328
+ "alert.assessment.impact.severity" => ["%{severity}", "%{level}"],
329
+ "alert.assessment.action.description" => ["%{action}"],
330
+ }
331
+
302
332
  private
303
333
  def idmefpaths_to_xml(event, paths, doc = nil)
334
+ # create the document if not existing
304
335
  if doc.nil?
305
336
  doc = Nokogiri::XML::Document.new
337
+ if @validate_xml
338
+ doc.create_external_subset('IDMEF-Message', nil, @dtd_path)
339
+ end
306
340
  doc.root = Nokogiri::XML::Node.new('IDMEF-Message', doc)
307
341
  doc.root.add_namespace_definition('idmef', 'http://iana.org/idmef')
308
342
  end
309
- event_to_remove = []
310
- paths.each do |path, value|
311
- if !value.kind_of?(Array)
312
- value = [value]
313
- end
314
- value.each do |v|
315
- if v.to_s.start_with?("$")
316
- c = ''
317
- f = true
318
- v[1..-1].split('.').each do |ppath|
319
- if !event.get(c + '[' + ppath + ']').nil?
320
- c = c + '[' + ppath + ']'
321
- else
322
- f = false
323
- end
324
- end
325
- if !f then next end
326
- value = event.get(c)
327
- event_to_remove << c
328
- else
329
- value = v
330
- end
331
- end
332
- if value.kind_of?(Array) or value.to_s.empty?
333
- next
343
+
344
+ # translate all path inot the xml
345
+ paths.each do |path, values|
346
+ if !values.kind_of?(Array)
347
+ values = [values]
334
348
  end
335
- if value.kind_of?(String)
336
- @utf8_charset.convert(value)
349
+
350
+ formated_value = nil
351
+ values.each do |value|
352
+ formated_value = event.sprintf(value)
353
+ # value is looking for non existing variable in event
354
+ if /%{[^}]+}/.match(formated_value).nil?
355
+ break
356
+ end
357
+
358
+ if formated_value == value
359
+ formated_value = nil
360
+ end
337
361
  end
338
- curr = doc.root
339
- rfc = IDMEFMessage
340
- path.split('.').each do |name|
341
- ret = name.match(/^(.*)\((\d+)\)/)
342
- if ret
343
- name = ret[1]
344
- v = (ret ? ret[2] : 0).to_i
362
+
363
+ next if formated_value.nil? or formated_value.empty?
364
+
365
+ @utf8_charset.convert(formated_value)
366
+
367
+ xml_current_node = doc.root
368
+ rfc_current_class = @@IDMEFMessage
369
+ # path is an idmef path. example : alert.classification.text
370
+ path.split('.').each do |idmefpath_name|
371
+ # handle listed_path like alert.target(0).node.address(0).address
372
+ listed_path = idmefpath_name.match(/^(.*)\((\d+)\)/)
373
+ idmefpath_index = nil
374
+ if listed_path
375
+ idmefpath_name = listed_path[1]
376
+ idmefpath_index = (listed_path ? listed_path[2] : 0).to_i
345
377
  end
346
378
 
347
- ne = rfc[name][:class]
348
- ne_t = rfc[name][:type]
349
- path_idmef = rfc[name][:type] == :list_class ? ne[:name] : nil
379
+ idmefpath_rfc_elm = rfc_current_class[idmefpath_name]
380
+
381
+ idmef_node_name = nil
382
+ if rfc_current_class[idmefpath_name][:type] == :list_class
383
+ idmef_node_name = idmefpath_rfc_elm[:class][:name]
384
+ end
350
385
 
351
- if ret && ne_t == :list_class
352
- c = curr.xpath(path_idmef)
353
- if c.empty?
354
- no = Nokogiri::XML::Node.new(ne[:name], doc)
355
- curr << no
356
- elsif c.length <= v
357
- nod = c[-1]
358
- (c.length..v).each do |t|
359
- no = Nokogiri::XML::Node.new(ne[:name], doc)
360
- nod.after(no)
361
- nod = no
386
+ # rfc class with multiple elements
387
+ if !idmefpath_index.nil? && idmefpath_rfc_elm[:type] == :list_class
388
+ idmef_nodes = xml_current_node.xpath(idmef_node_name)
389
+
390
+ if idmef_nodes.empty?
391
+ idmef_node = Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
392
+ xml_current_node << idmef_node
393
+
394
+ elsif idmef_nodes.length <= idmefpath_index
395
+ idmef_node = idmef_nodes[-1]
396
+ (idmef_nodes.length..idmefpath_index).each do |idx|
397
+ tmp_node = Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
398
+ idmef_node.after(tmp_node)
399
+ idmef_node = tmp_node
362
400
  end
363
- elsif c.length > v
364
- no = c[v]
401
+
402
+ elsif idmef_nodes.length > idmefpath_index
403
+ idmef_node = idmef_nodes[idmefpath_index]
365
404
  end
366
- curr = no
367
- rfc = ne
368
- elsif !ret && ne_t == :list_class
369
- no = curr.xpath(path_idmef).first || Nokogiri::XML::Node.new(ne[:name], doc)
370
- curr << no
371
- curr = no
372
- rfc = ne
373
- elsif ne_t == :list_value
374
- if rfc[name][:format] == :datetime
375
- value = DateTime.parse(value.to_s).strftime("%FT%T%:z")
405
+
406
+ xml_current_node = idmef_node
407
+ rfc_current_class = idmefpath_rfc_elm[:class]
408
+
409
+ # rfc class with on element
410
+ elsif idmefpath_index.nil? && idmefpath_rfc_elm[:type] == :list_class
411
+ idmef_node = xml_current_node.xpath(idmef_node_name).first
412
+ idmef_node = idmef_node || Nokogiri::XML::Node.new(idmefpath_rfc_elm[:class][:name], doc)
413
+ xml_current_node << idmef_node
414
+ xml_current_node = idmef_node
415
+ rfc_current_class = idmefpath_rfc_elm[:class]
416
+
417
+ # rfc multiple values
418
+ elsif idmefpath_rfc_elm[:type] == :list_value
419
+ if rfc_current_class[idmefpath_name][:name] == :type
420
+ node_name = xml_current_node["type"]
421
+ else
422
+ node_name = rfc_current_class[idmefpath_name][:name]
376
423
  end
377
- n = rfc[name][:name] == :type ? curr["type"] : rfc[name][:name]
378
- no = Nokogiri::XML::Node.new(n, doc)
379
- no.content = value.to_s
380
- curr << no
381
- elsif ne_t == :attr
382
- if rfc[name][:format] == :datetime
383
- value = DateTime.parse(value.to_s).strftime("%FT%T%:z")
424
+
425
+ idmef_node = Nokogiri::XML::Node.new(node_name, doc)
426
+
427
+ # reformat datetime with the expected format described in idmef rfc
428
+ if rfc_current_class[idmefpath_name][:format] == :datetime
429
+ alert_time = DateTime.parse(formated_value)
430
+ formated_value = alert_time.strftime(@@IDMEF_Time_Format)
431
+ seconds = alert_time.to_time.to_i + 2208988800
432
+ seconds_fraction = (alert_time.to_time.usec. / (1000000.0 / (2 ** 32))).to_i
433
+ idmef_node["ntpstamp"] = "0x%08x.0x%08x" % [seconds, seconds_fraction]
384
434
  end
385
- curr[rfc[name][:name]] = value.to_s
435
+
436
+ idmef_node.content = formated_value
437
+
438
+ xml_current_node << idmef_node
439
+
440
+ # rfc attribute
441
+ elsif idmefpath_rfc_elm[:type] == :attr
442
+ xml_current_node[rfc_current_class[idmefpath_name][:name]] = formated_value.to_s
443
+
386
444
  end
387
- rfc.each do |kk, vv|
388
- if vv.respond_to?(:each_pair) && vv[:default] && vv[:type] == :attr && !curr[vv[:name]]
389
- curr[vv[:name]] = vv[:default]
445
+
446
+ # set default values as described in rfc
447
+ rfc_current_class.each do |element, value|
448
+ # value is a ref, a string or a hash, we want hashs
449
+ next if !value.respond_to?(:each_pair)
450
+
451
+ if value[:default] && value[:type] == :attr && !xml_current_node[value[:name]]
452
+ xml_current_node[value[:name]] = value[:default]
390
453
  end
391
454
  end
392
455
  end
393
456
  end
394
- event_to_remove.each do |v|
395
- event.remove(v)
396
- end
397
457
  return doc
398
458
  end
399
459
 
400
460
  private
401
461
  def xml_to_string(doc)
402
- doc.root.traverse { |node|
462
+ # add namespace "idmef"
463
+ doc.root.traverse do |node|
403
464
  if node.type != Nokogiri::XML::Node::TEXT_NODE
404
465
  node.name = 'idmef:' + node.name
405
466
  end
406
- }
467
+ end
468
+
469
+ # return a oneline xml without spaces
407
470
  return doc.serialize(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML).sub("\n", "").strip
408
471
  end
409
472
 
@@ -413,75 +476,93 @@ class LogStash::Codecs::IDMEF < LogStash::Codecs::Base
413
476
  @utf8_charset = LogStash::Util::Charset.new('UTF-8')
414
477
  @utf8_charset.logger = self.logger
415
478
 
416
- @local_paths = {
417
- "alert.analyzer(0).name" => ["$product", "$devname"],
418
- "alert.analyzer(0).manufacturer" => "$vendor",
419
- "alert.create_time" => "$@timestamp",
420
- "alert.detect_time" => "$@timestamp",
421
- "alert.analyzer_time" => "$@timestamp",
422
- "alert.source(0).node.address(0).address" => ["$srcip", "$src"],
423
- "alert.source(0).node.name" => ["$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"],
424
- "alert.source(0).service.port" => ["$spt", "$sport", "$s_port"],
425
- "alert.source(0).service.name" => ["$sservice", "$srcservice"],
426
- "alert.target(0).node.address(0).address" => ["$hostip", "$dstip", "$dst", "$ip"],
427
- "alert.target(0).node.name" => ["$host", "$hostname", "$shost", "$srchost", "$shostname", "$srchostname", "$sname", "$srcname"],
428
- "alert.target(0).service.port" => ["$dpt", "$dport", "$d_port"],
429
- "alert.target(0).service.name" => ["$service", "$service_id", "$dservice", "$dstservice",],
430
- "alert.target(0).user.user_id(0).name" => ["$user", "$dstuser", "$duser"],
431
- "alert.target(0).user.user_id(0).number" => ["$uid", "$dstuid", "$duid"],
432
- "alert.target(0).process.name" => ["$proc", "$process"],
433
- "alert.target(0).process.pid" => ["$dpid", "$pid"],
434
- "alert.classification.text" => ["$rule_name", "$event", "$message"],
435
- "alert.assessment.impact.severity" => ["$severity", "$level"],
436
- "alert.assessment.action.description" => ["$action"],
437
- }
438
479
  if @defaults
439
- @allpaths = @local_paths.merge(@paths)
480
+ @allpaths = @@local_paths.merge(@paths)
440
481
  else
441
482
  @allpaths = @paths
442
483
  end
484
+
485
+ if @additionaldata
486
+ # Find all event's keys already used in @@IDMEF paths values
487
+ @allpaths_event_keys = []
488
+ @allpaths.each do |key, values|
489
+ if !values.kind_of?(Array)
490
+ values = [values]
491
+ end
492
+
493
+ values.each do |value|
494
+ match = value.match(/%{([^}]+)}/)
495
+ if match
496
+ @allpaths_event_keys += match.captures
497
+ end
498
+ end
499
+ end
500
+ end
501
+
502
+ if @validate_xml
503
+ @dtd_path = File.dirname(File.expand_path(__FILE__)) + "/idmef-message.dtd"
504
+ @dtd_options = Nokogiri::XML::ParseOptions.new()
505
+ @dtd_options.recover
506
+ @dtd_options.dtdload
507
+ @dtd_options.dtdvalid
508
+ end
443
509
  end
444
510
 
445
511
  public
446
512
  def encode(event)
447
- # Reload configuration
448
- @allpaths = @allpaths.merge(@paths)
513
+ # Set messageid and analyzerid
514
+ paths = { "%s.messageid" % @type => java.util.UUID.randomUUID.to_s,
515
+ "%s.analyzer(0).analyzerid" % @type => Socket.gethostname.to_s
516
+ }
449
517
 
450
- # Copy event
451
- e = event.clone
518
+ # CreateTime is required in IDMEF RFC
519
+ if !@allpaths.include? "alert.create_time"
520
+ paths["alert.create_time"] = DateTime.now().strftime(@@IDMEF_Time_Format)
521
+ end
452
522
 
453
- # Set messageid and analyzerid
454
- p = { "%s.messageid" % @type => java.util.UUID.randomUUID.to_s,
455
- "%s.analyzer(0).analyzerid" % @type => Socket.gethostname.to_s
456
- }
457
- xml = idmefpaths_to_xml(e, p)
523
+ # Classification is required in IDMEF RFC
524
+ if !@allpaths.include? "alert.classification.text"
525
+ paths["alert.classification.text"] = "Unknown alert"
526
+ end
527
+
528
+ xml = idmefpaths_to_xml(event, paths)
458
529
 
459
- # Set paths
460
- xml = idmefpaths_to_xml(e, @allpaths, xml)
530
+ # Set configured paths
531
+ xml = idmefpaths_to_xml(event, @allpaths, xml)
461
532
 
462
- # Set Additional data
533
+ # Add unused event data to IDMEF additional data
463
534
  if @additionaldata
464
- idx = xml.xpath('/IDMEF-Message/Alert/AddionnalData').length
465
- e.to_hash.each do |key, value|
466
- if value.to_s.empty?
467
- next
468
- end
535
+ additionaldata_idx = xml.xpath('/idmef-message/alert/addionnaldata').length
536
+
537
+ event.to_hash.each do |key, value|
538
+ next if value.to_s.empty? or @allpaths_event_keys.include? key
539
+
469
540
  if value.kind_of?(Integer)
470
- t = "integer"
541
+ value_type = "integer"
471
542
  elsif value.kind_of?(Float)
472
- t = "real"
543
+ value_type = "real"
473
544
  else
474
- t = "string"
545
+ value_type = "string"
475
546
  end
476
- p = { "alert.additional_data(%d).meaning" % idx => key,
477
- "alert.additional_data(%d).type" % idx => t,
478
- "alert.additional_data(%d).data" % idx => value.to_s,
479
- }
480
- xml = idmefpaths_to_xml(e, p , xml)
481
- idx = idx + 1
547
+
548
+ paths = { "alert.additional_data(%d).meaning" % additionaldata_idx => key,
549
+ "alert.additional_data(%d).type" % additionaldata_idx => value_type,
550
+ "alert.additional_data(%d).data" % additionaldata_idx => value.to_s,
551
+ }
552
+
553
+ xml = idmefpaths_to_xml(event, paths , xml)
554
+ additionaldata_idx += 1
482
555
  end
483
556
  end
484
557
 
558
+ if @validate_xml
559
+ xml_dtd = Nokogiri::XML.parse(xml.to_xml, nil, nil, @dtd_options)
560
+ if !xml_dtd.validate.nil? and !xml_dtd.validate.empty?
561
+ raise "IDMEF XML generated is not valid. Errors: %s." % xml_dtd.validate.join(', ')
562
+ end
563
+ xml.external_subset.remove
564
+ end
565
+
485
566
  # Create the XML
486
567
  @on_event.call(event, xml_to_string(xml) + NL)
487
568
  end