unified2 0.5.4 → 0.6.0

data/ChangeLog.md

@@ -1,3 +1,13 @@
+=== 0.6.0 / 2011-11-13
+
+* update deps
+* added support for unified2 extra data
+* refactor Unified2#read & Unified2#watch
+* Interrupt now returns file position
+* updated spec for legacy u2 and current format changes
+* events can now have multiple packets
+* bug fixes and documentation
+
 === 0.5.4 / 2011-06-27
 
 * update packetfu ~> 1.1

data/README.md

@@ -1,4 +1,4 @@
-# unified2
+# Unified2
 
 * [Homepage](http://github.com/mephux/unified2)
 * [Issues](http://github.com/mephux/unified2/issues)
@@ -12,57 +12,63 @@ A ruby interface for unified2 output. rUnified2 allows you to manipulate unified
 ## Features
 
  * Monitor/Read unified2 logs & manipulate the data.
- * Numerous connivence methods
+ * Numerous convenience methods
  * Simple & Intuitive to Use
+ * Supports legacy unified2 formats and the most current as of snort 2.9.1.3
+ * Packet data, headers, hexdumps and more.
 
 ## Examples
 
-``` ruby
-require 'unified2'
+    require 'unified2'
 
-#
-# Load rules into memory
-#
+    # Unified2 Configuration
+    Unified2.configuration do
 
-Unified2.configuration do
- # Sensor Configurations
- sensor :id => 1, :name => 'Test Sensor', :interface => 'en1'
+      # Sensor Configurations
+      sensor :interface => 'en1',
+        :name => 'Unified2 Example', :id => 3
 
- # Load signatures, generators & classifications into memory
- load :signatures, 'sid-msg.map'
- load :generators, 'gen-msg.map'
- load :classifications, 'classification.config'
-end
+      load :signatures, 'seeds/sid-msg.map'
 
-#
-# Unified2#watch
-#
-# Watch a unified2 file for changes and process the results.
-# 
+      load :generators, 'seeds/gen-msg.map'
+      
+      load :classifications, 'seeds/classification.config'
 
-Unified2.watch('/var/log/snort/merged.log', :last) do |event|
- next if event.signature.name.blank?
- puts event	
-end
+    end
 
-# Unified2#read
-# Parse a unified2 file and process the results.
+    Unified2.watch('seeds/unified2-current.log', :first) do |event|
 
-Unified2.read('/var/log/snort/merged.log') do |event|
+      puts event.id
 
- puts event.protocol #=> "TCP"
+      puts event.severity
 
- puts event.protocol.to_h #=> {:length=>379, :seq=>3934511163, :ack=>1584708129 ... }
+      puts event.classification.name
 
-end
-```
+      puts event.signature.name
+
+      event.extras.each do |extra|
+        puts extra.name
+        puts extra.value
+      end
+
+      event.packets.each do |packet|
+        puts packet.ip_header
+        puts packet.protocol.header
+        puts packet.hexdump(:header => false, :width => 40)
+      end
+
+    end
 
 ## Requirements
 
- * bindata ~> 1.3.1
- * hexdump: ~> 0.1.0
- * packetfu: ~> 1.0.0
- * pcaprub: ~> 0.9.2
+ * bindata ~> 1.4.x
+ * hexdump: ~> 0.2.x
+ * packetfu: ~> 1.1.x
+
+## TODO
+
+ * Make both Event#watch and Event#read evented
+ * User eventmachine to monitor the file i.e modify/delete/move/symlink
 
 ## Install
 

data/Rakefile

@@ -1,3 +1,4 @@
+require 'psych'
 require 'rubygems'
 require 'rake'
 
@@ -26,7 +27,7 @@ task :test => :spec
 task :default => :spec
 
 begin
-  gem 'yard', '~> 0.6.0'
+  gem 'yard', '~> 0.7'
   require 'yard'
 
   YARD::Rake::YardocTask.new  
@@ -36,4 +37,4 @@ rescue LoadError => e
   end
 end
 
-task :doc => :yard
+task :doc => :yard

data/bin/ru2

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+require 'unified2'
+
+path = ARGV.first
+
+Unified2.configuration do
+  sensor :interface => 'en1',
+    :name => 'Unified2 Example', :id => 0
+end
+
+
+def render(event)
+  data = "EVENT\n"
+  data += "\tevent id: #{event.id}\n"
+  data += "\tsensor id: #{event.sensor.id}\n"
+  data += "\ttimestamp: #{event.timestamp.strftime('%D %H:%M:%S')}\n"
+  data += "\tseverity: #{event.severity}\n"
+  data += "\tprotocol: #{event.protocol}\n"
+  data += "\tsource ip: #{event.source_ip} (#{event.source_port})\n"
+  data += "\tdestination ip: #{event.destination_ip} (#{event.destination_port})\n"
+  data += "\tsignature: #{event.signature.id}\n"
+  data += "\tclassification: #{event.classification.id}\n"
+  data += "\tchecksum: #{event.checksum}\n"
+
+  packet_count = 1
+  length = event.packets.count
+
+  event.packets.each do |packet|
+    data += "\n\tPACKET  (#{packet_count} of #{length})\n\n"
+
+    data += "\tsensor id: #{event.sensor.id}"
+    data += "\tevent id: #{event.id}"
+    data += "\tevent second: #{packet.event_timestamp.to_i}\n"
+    data += "\tpacket second: #{packet.timestamp.to_i}"
+    data += "\tpacket microsecond: #{packet.microsecond.to_i}\n"
+    data += "\tlinktype: #{packet.link_type}"
+    data += "\tpacket length: #{packet.length}\n"
+    data += "\tchecksum: #{packet.checksum}\n\n"
+
+    hexdump = packet.hexdump(:width => 16)
+    hexdump.each_line { |line| data += "\t" + line }
+
+    packet_count += 1
+  end
+
+  extra_count = 1
+  length = event.extras.count
+
+  event.extras.each do |extra|
+    data += "\n\tEXTRA   (#{extra_count} of #{length})\n\n"
+
+    data += "\tname: #{extra.name}"
+    data += "\tevent type: #{extra.header[:event_type]}"
+    data += "\tevent length: #{extra.header[:event_length]}\n"
+    data += "\tsensor id: #{event.sensor.id}"
+    data += "\tevent id: #{event.id}"
+    data += "\tevent second: #{extra.timestamp}\n"
+    data += "\ttype: #{extra.type_id}"
+    data += "\tdata type: #{extra.data_type}"
+    data += "\tlength: #{extra.length}\n"
+    data += "\tvalue: " + extra.value + "\n"
+
+    extra_count += 1
+  end
+
+  data += "\n"
+end
+
+unless path
+  STDERR.puts "You must supply a unified2 log file."
+  exit 1
+end
+
+Unified2.read(path) do |event|
+  puts render(event)
+end

data/example/example.rb

@@ -1,33 +1,25 @@
-$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+$:<< '../lib' << 'lib'
+
 require 'unified2'
 
 # Unified2 Configuration
 Unified2.configuration do
-  
+
   # Sensor Configurations
-  sensor :interface => 'en1', 
-  :name => 'Example Sensor', :id => 3
+  sensor :interface => 'en1',
+    :name => 'Unified2 Example', :id => 3
 
-  # Load signatures, generators & classifications into memory
   load :signatures, 'seeds/sid-msg.map'
 
   load :generators, 'seeds/gen-msg.map'
   
   load :classifications, 'seeds/classification.config'
-  
+
 end
 
-#
-# Monitor the unfied2 log and process the data.
-# 
-# The second argument is the last event processed by
-# the sensor. If the last_event_id column is blank in the
-# sensor table it will begin at the first available event.
-#
-Unified2.watch('seeds/unified2.log', :first) do |event|
-  next if event.signature.blank?
-  
+Unified2.watch('seeds/unified2-current.log', :first) do |event|
+
   puts event
-  puts "\n"
-  
+
 end
+

data/example/example2.rb

@@ -0,0 +1,44 @@
+$:<< '../lib' << 'lib'
+
+require 'unified2'
+
+# Unified2 Configuration
+Unified2.configuration do
+
+  # Sensor Configurations
+  sensor :interface => 'en1',
+    :name => 'Unified2 Example', :id => 3
+
+  # Load signatures, generate events will be sent over the web socket
+  # quickly so we slow down the process of
+  # pushing events onto the channel.rs & classifications into memory
+  load :signatures, 'seeds/sid-msg.map'
+
+  load :generators, 'seeds/gen-msg.map'
+  
+  load :classifications, 'seeds/classification.config'
+
+end
+
+Unified2.watch('seeds/unified2-current.log', :first) do |event|
+  
+  puts event.id
+
+  puts event.severity
+
+  puts event.classification.name
+
+  puts event.signature.name
+
+  event.extras.each do |extra|
+    puts extra.name
+    puts extra.value
+  end
+
+  event.packets.each do |packet|
+    puts packet.ip_header
+    puts packet.protocol.header
+    puts packet.hexdump(:header => false, :width => 40)
+  end
+
+end

data/example/seeds/classification.config

@@ -1,4 +1,4 @@
-# $Id: classification.config,v 1.4 2010/04/15 19:53:02 mwatchinski Exp $
+# $Id: classification.config,v 1.4 2010-04-15 19:53:02 mwatchinski Exp $
 # The following includes information for prioritizing rules
 # 
 # Each classification includes a shortname, a description, and a default

data/example/seeds/gen-msg.map

@@ -1,4 +1,4 @@
-# $Id: gen-msg.map,v 1.10 2010/07/07 15:51:00 nhoughton Exp $
+# $Id: gen-msg.map,v 1.14 2011-10-07 20:21:24 nhoughton Exp $
 # GENERATORS -> msg map
 # Format: generatorid || alertid || MSG
 
@@ -40,7 +40,7 @@
 111 || 5 || spp_stream4: Data on SYN Packet
 111 || 6 || spp_stream4: Full XMAS Stealth Scan
 111 || 7 || spp_stream4: SAPU Stealth Scan
-111 || 8 || spp_stream4: FIN Stealth Scan 
+111 || 8 || spp_stream4: FIN Stealth Scan
 111 || 9 || spp_stream4: NULL Stealth Scan
 111 || 10 || spp_stream4: NMAP XMAS Stealth Scan
 111 || 11 || spp_stream4: VECNA Stealth Scan
@@ -163,6 +163,10 @@
 116 || 292 || snort_decoder: WARNING: IPv6 header has destination options followed by a routing header
 116 || 293 || snort_decoder: WARNING: Two or more IP (v4 and/or v6) encapsulation layers present
 116 || 294 || snort_decoder: WARNING: truncated Encapsulated Security Payload (ESP) header
+116 || 295 || snort_decoder: WARNING: IPv6 header includes an option which is too big for the containing header.
+116 || 296 || snort_decoder: WARNING: IPv6 packet includes out-of-order extension headers
+116 || 297 || snort_decoder: WARNING: Two or more GTP encapsulation layers are present
+116 || 298 || snort_decoder: WARNING: GTP header length is invalid
 116 || 400 || snort_decoder: WARNING: XMAS Attack Detected!
 116 || 401 || snort_decoder: WARNING: Nmap XMAS Attack Detected!
 116 || 402 || snort_decoder: WARNING: DOS NAPTHA Vulnerability Detected!
@@ -215,6 +219,11 @@
 116 || 449 || snort_decoder: WARNING: BAD-TRAFFIC Unassigned/Reserved IP protocol
 116 || 450 || snort_decoder: WARNING: BAD-TRAFFIC Bad IP protocol
 116 || 451 || snort_decoder: WARNING: ICMP PATH MTU denial of service attempt
+116 || 452 || snort_decoder: WARNING: BAD-TRAFFIC linux ICMP header dos attempt
+116 || 453 || snort_decoder: WARNING: IPV6 ISATAP spoof
+116 || 454 || snort_decoder: WARNING: PGM NAK overflow
+116 || 455 || snort_decoder: WARNING: IGMP options dos
+116 || 456 || snort_decoder: WARNING: too many IPV6 extension headers
 117 || 1 || spp_portscan2: Portscan detected!
 118 || 1 || spp_conversation: Bad IP protocol!
 119 || 1 || http_inspect: ASCII ENCODING
@@ -240,8 +249,22 @@
 119 || 21 || http_inspect: MULTIPLE CONTENT LENGTH HEADER FIELDS
 119 || 22 || http_inspect: CHUNK SIZE MISMATCH DETECTED
 119 || 23 || http_inspect: INVALID IP IN TRUE-CLIENT-IP/XFF HEADER
+119 || 24 || http_inspect: MULTIPLE HOST HEADERS DETECTED 
+119 || 25 || http_inspect: HOSTNAME EXCEEDS 255 CHARACTERS
+119 || 26 || http_inspect: HEADER PARSING SPACE SATURATION
+119 || 27 || http_inspect: CHUNKED ENCODING - EXCESSIVE CONSECUTIVE SMALL CHUNKS
+119 || 28 || http_inspect: POST W/O CONTENT-LENGTH OR CHUNKS
 120 || 1 || http_inspect: ANOMALOUS HTTP SERVER ON UNDEFINED HTTP PORT
 120 || 2 || http_inspect: INVALID STATUS CODE IN HTTP RESPONSE
+120 || 3 || http_inspect: NO CONTENT-LENGTH OR TRANSFER-ENCODING IN HTTP RESPONSE
+120 || 4 || http_inspect: HTTP RESPONSE HAS UTF CHARSET WHICH FAILED TO NORMALIZE
+120 || 5 || http_inspect: HTTP RESPONSE HAS UTF-7 CHARSET
+120 || 6 || http_inspect: HTTP RESPONSE GZIP DECOMPRESSION FAILED
+120 || 7 || http_inspect: CHUNKED ENCODING - EXCESSIVE CONSECUTIVE SMALL CHUNKS
+120 || 8 || http_inspect: MESSAGE WITH INVALID CONTENT-LENGTH OR CHUNK SIZE
+120 || 9 || http_inspect: JAVASCRIPT OBFUSCATION LEVELS EXCEEDS 2
+120 || 10 || http_inspect: JAVASCRIPT WHITESPACES EXCEEDS MAX ALLOWED
+120 || 11 || http_inspect: MULTIPLE ENCODINGS WITHIN JAVASCRIPT OBFUSCATION FUNCS
 121 || 1 || flow-portscan: Fixed Scale Scanner Limit Exceeded
 121 || 2 || flow-portscan: Sliding Scale Scanner Limit Exceeded
 121 || 3 || flow-portscan: Fixed Scale Talker Limit Exceeded
@@ -294,6 +317,11 @@
 124 || 6 || smtp: Illegal command
 124 || 7 || smtp: Attempted header name buffer overflow
 124 || 8 || smtp: Attempted X-Link2State command buffer overflow
+124 || 9 || smtp: No memory available for decoding. Max Mime Mem exceeded.
+124 || 10 || smtp: Base64 Decoding failed
+124 || 11 || smtp: Quoted-Printable Decoding failed
+124 || 12 || smtp: 7bit/8bit/binary/text Extraction failed
+124 || 13 || smtp: Unix-to-Unix Decoding failed
 125 || 1 || ftp_pp: Telnet command on FTP command channel
 125 || 2 || ftp_pp: Invalid FTP command
 125 || 3 || ftp_pp: FTP parameter length overflow
@@ -306,8 +334,8 @@
 126 || 1 || telnet_pp: Telnet consecutive AYT overflow
 126 || 2 || telnet_pp: Telnet data encrypted
 126 || 3 || telnet_pp: Subnegotiation Begin without matching Subnegotiation End
-128 || 1 || ssh: Gobbles exploit 
-128 || 2 || ssh: SSH1 CRC32 exploit 
+128 || 1 || ssh: Gobbles exploit
+128 || 2 || ssh: SSH1 CRC32 exploit
 128 || 3 || ssh: Server version string overflow
 128 || 4 || ssh: Protocol mismatch
 128 || 5 || ssh: Bad message direction
@@ -323,10 +351,15 @@
 129 || 8 || stream5: Data sent on stream after TCP Reset
 129 || 9 || stream5: TCP Client possibly hijacked, different Ethernet Address
 129 || 10 || stream5: TCP Server possibly hijacked, different Ethernet Address
-129 || 11 || stream5: TCP Data with no TCP Flags set 
+129 || 11 || stream5: TCP Data with no TCP Flags set
 129 || 12 || stream5: TCP Small Segment Threshold Exceeded
 129 || 13 || stream5: TCP 4-way handshake detected
 129 || 14 || stream5: TCP Timestamp is missing
+129 || 15 || stream5: Reset outside window
+129 || 16 || stream5: FIN number is greater than prior FIN
+129 || 17 || stream5: ACK number is greater than prior FIN
+129 || 18 || stream5: Data sent on stream after TCP Reset received
+129 || 19 || stream5: TCP window closed before receiving data
 130 || 1 || dcerpc: Maximum memory usage reached
 131 || 1 || dns: Obsolete DNS RData Type
 131 || 2 || dns: Experimental DNS RData Type
@@ -374,18 +407,62 @@
 133 || 41 || dcerpc2: Connectionless DCE/RPC - Invalid pdu type
 133 || 42 || dcerpc2: Connectionless DCE/RPC - Data length less than header size
 133 || 43 || dcerpc2: Connectionless DCE/RPC - Bad sequence number
-133 || 44 || dcerpc2: SMB - Invalid SMB version 1 seen
-133 || 45 || dcerpc2: SMB - Invalid SMB version 2 seen
-133 || 46 || dcerpc2: SMB - Invalid user, tree connect, file binding
-133 || 47 || dcerpc2: SMB - Excessive command compounding
+#133 || 44 || dcerpc2: SMB - Invalid SMB version 1 seen
+#133 || 45 || dcerpc2: SMB - Invalid SMB version 2 seen
+#133 || 46 || dcerpc2: SMB - Invalid user, tree connect, file binding
+#133 || 47 || dcerpc2: SMB - Excessive command compounding
 134 || 1 || ppm: rule tree disabled
 134 || 2 || ppm: rule tree enabled
 135 || 1 || internal: syn received
 135 || 2 || internal: session established
 135 || 3 || internal: session cleared
+136 || 1 || reputation: Packet is blacklisted
+136 || 2 || reputation: Packet is whitelisted
+137 || 1 || ssp_ssl: Invalid Client HELLO after Server HELLO Detected
+137 || 2 || ssp_ssl: Invalid Server HELLO without Client HELLO Detected
 138 || 2 || sensitive_data: sensitive data - Credit card numbers
 138 || 3 || sensitive_data: sensitive data - U.S. social security numbers with dashes
 138 || 4 || sensitive_data: sensitive data - U.S. social security numbers without dashes
 138 || 5 || sensitive_data: sensitive data - eMail addresses
 138 || 6 || sensitive_data: sensitive data - U.S. phone numbers
 139 || 1 || sensitive_data: sensitive data global threshold exceeded
+140 || 1 || sip: Maximum sessions reached 
+140 || 2 || sip: Empty request URI 
+140 || 3 || sip: URI is too long
+140 || 4 || sip: Empty call-Id
+140 || 5 || sip: Call-Id is too long
+140 || 6 || sip: CSeq number is too large or negative
+140 || 7 || sip: Request name in CSeq is too long 
+140 || 8 || sip: Empty From header
+140 || 9 || sip: From header is too long
+140 || 10 || sip: Empty To header
+140 || 11 || sip: To header is too long
+140 || 12 || sip: Empty Via header 
+140 || 13 || sip: Via header is too long
+140 || 14 || sip: Empty Contact
+140 || 15 || sip: Contact is too long
+140 || 16 || sip: Content length is too large or negative
+140 || 17 || sip: Multiple SIP messages in a packet
+140 || 18 || sip: Content length mismatch
+140 || 19 || sip: Request name is invalid
+140 || 20 || sip: Invite replay attack
+140 || 21 || sip: Illegal session information modification
+140 || 22 || sip: Response status code is not a 3 digit number
+140 || 23 || sip: Empty Content type
+140 || 24 || sip: SIP version other than 2.0, 1.0, and 1.1 are invalid
+140 || 25 || sip: Mismatch in Method of request and the CSEQ header
+140 || 26 || sip: The method is unknown
+141 || 1 || imap: Unknown IMAP4 command
+141 || 2 || imap: Unknown IMAP4 response
+141 || 3 || imap: No memory available for decoding. Memcap exceeded.
+141 || 4 || imap: Base64 Decoding failed
+141 || 5 || imap: Quoted-Printable Decoding failed
+141 || 6 || imap: 7bit/8bit/binary/text Extraction failed
+141 || 7 || imap: Unix-to-Unix Decoding failed
+142 || 1 || pop: Unknown POP3 command
+142 || 2 || pop: Unknown POP3 response
+142 || 3 || pop: No memory available for decoding. Memcap exceeded.
+142 || 4 || pop: Base64 Decoding failed
+142 || 5 || pop: Quoted-Printable Decoding failed
+142 || 6 || pop: 7bit/8bit/binary/text Extraction failed
+142 || 7 || pop: Unix-to-Unix Decoding failed