woolen_common 0.0.1

data/lib/woolen_common/pcap/mu/pcap/pkthdr.rb

@@ -0,0 +1,162 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+    class Pcap
+
+        class Pkthdr
+            attr_accessor :endian, :ts_sec, :ts_usec, :caplen, :len, :pkt, :pkt_raw
+
+            def initialize endian=BIG_ENDIAN, ts_sec=0, ts_usec=0, caplen=0, len=0, pkt=nil
+                @endian = endian
+                @ts_sec = ts_sec
+                @ts_usec = ts_usec
+                @caplen = caplen
+                @len = len
+                @pkt = pkt
+                @pkt_raw = pkt
+            end
+
+            FMT_NNNN = 'NNNN'
+            FMT_VVVV = 'VVVV'
+
+            def self.read io, endian=BIG_ENDIAN
+                if endian == BIG_ENDIAN
+                    format = FMT_NNNN
+                elsif endian == LITTLE_ENDIAN
+                    format = FMT_VVVV
+                end
+                bytes = io.read 16
+                if not bytes
+                    raise EOFError, 'Missing PCAP packet header'
+                end
+                if not bytes.length == 16
+                    raise ParseError, "Truncated PCAP packet header: expected 16 bytes, got #{bytes.length} bytes"
+                end
+                ts_sec, ts_usec, caplen, len = bytes.unpack format
+                pkt = io.read(caplen)
+                if not pkt
+                    raise ParseError, 'Missing PCAP packet header packet'
+                end
+                if not pkt.length == caplen
+                    raise ParseError, "Truncated PCAP packet header: expected #{pkthdr.caplen} bytes, got #{pkthdr.pkt.length} bytes"
+                end
+                pkthdr = Pkthdr.new endian, ts_sec, ts_usec, caplen, len, pkt
+                return pkthdr
+            end
+
+            def write io
+                if @pkt.is_a? String
+                    pkt = @pkt
+                else
+                    string_io = StringIO.new
+                    @pkt.write string_io
+                    pkt = string_io.string
+                end
+                len = pkt.length
+                bytes = [@ts_sec, @ts_usec, len, len].pack FMT_NNNN
+                io.write bytes
+                io.write pkt
+            end
+
+            def decode! endian, linktype
+                @pkt = case linktype
+                           when DLT_NULL;
+                               Pkthdr.decode_null endian, @pkt
+                           when DLT_EN10MB;
+                               Pkthdr.decode_en10mb @pkt
+                           when DLT_RAW;
+                               raise NotImplementedError
+                           when DLT_LINUX_SLL;
+                               Pkthdr.decode_linux_sll @pkt
+                           else
+                               raise ParseError, "Unknown PCAP linktype: #{linktype}"
+                       end
+            end
+
+            # See http://wiki.wireshark.org/NullLoopback
+            # and epan/aftypes.h in wireshark code.
+            BSD_AF_INET6 = [
+                OPENBSD_AF_INET6 = 24,
+                FREEBSD_AF_INET6 = 28,
+                DARWIN_AF_INET6 = 30
+            ]
+
+            def self.decode_null endian, bytes
+                Pcap.assert bytes.length >= 4, 'Truncated PCAP packet header: ' +
+                    "expected at least 4 bytes, got #{bytes.length} bytes"
+                if endian == BIG_ENDIAN
+                    format = 'N'
+                elsif endian == LITTLE_ENDIAN
+                    format = 'V'
+                end
+                family = bytes[0, 4].unpack(format)[0]
+                bytes = bytes[4..-1]
+                ethernet = Ethernet.new
+                ethernet.src = '00:01:01:00:00:01'
+                ethernet.dst = '00:01:01:00:00:02'
+                ethernet.payload = ethernet.payload_raw = bytes
+                if family != Socket::AF_INET and family != Socket::AF_INET6 and
+                    not BSD_AF_INET6.include?(family)
+                    raise ParseError, "Unknown PCAP packet header family: #{family}"
+                end
+                begin
+                    case family
+                        when Socket::AF_INET
+                            ethernet.type = Ethernet::ETHERTYPE_IP
+                            ethernet.payload = IPv4.from_bytes ethernet.payload
+                        when Socket::AF_INET6, FREEBSD_AF_INET6, OPENBSD_AF_INET6, DARWIN_AF_INET6
+                            ethernet.type = Ethernet::ETHERTYPE_IP6
+                            ethernet.payload = IPv6.from_bytes ethernet.payload
+                        else
+                            raise NotImplementedError
+                    end
+                rescue ParseError => e
+                    Pcap.warning e
+                end
+                return ethernet
+            end
+
+            def self.decode_en10mb bytes
+                return Ethernet.from_bytes(bytes)
+            end
+
+            def self.decode_linux_sll bytes
+                Pcap.assert bytes.length >= 16, 'Truncated PCAP packet header: ' +
+                    "expected at least 16 bytes, got #{bytes.length} bytes"
+                packet_type, link_type, addr_len = bytes.unpack('nnn')
+                type = bytes[14, 2].unpack('n')[0]
+                bytes = bytes[16..-1]
+                ethernet = Ethernet.new
+                ethernet.type = type
+                ethernet.src = '00:01:01:00:00:01'
+                ethernet.dst = '00:01:01:00:00:02'
+                ethernet.payload = ethernet.payload_raw = bytes
+                begin
+                    case type
+                        when Ethernet::ETHERTYPE_IP
+                            ethernet.payload = IPv4.from_bytes ethernet.payload
+                        when Ethernet::ETHERTYPE_IP6
+                            ethernet.payload = IPv6.from_bytes ethernet.payload
+                    end
+                rescue ParseError => e
+                    Pcap.warning e
+                end
+                return ethernet
+            end
+
+            def == other
+                return self.class == other.class &&
+                    self.endian == other.endian &&
+                    self.ts_sec == other.ts_sec &&
+                    self.ts_usec == other.ts_usec &&
+                    self.caplen == other.caplen &&
+                    self.len == other.len &&
+                    self.pkt == other.pkt
+            end
+        end
+
+    end
+end

data/lib/woolen_common/pcap/mu/pcap/reader.rb

@@ -0,0 +1,62 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+    class Pcap
+
+        class Reader
+            attr_accessor :pcap2scenario
+
+            FAMILY_TO_READER = {}
+
+            # Returns a reader instance of specified family. Returns nil when family is :none.
+            def self.reader family
+                if family == :none
+                    return nil
+                end
+
+                if klass = FAMILY_TO_READER[family]
+                    return klass.new
+                end
+
+                raise ArgumentError, "Unknown protocol family: '#{family}'"
+            end
+
+            # Returns family name
+            def family
+                raise NotImplementedError
+            end
+
+            # Notify parser of bytes written. Parser may update state
+            # to serve as a hint for subsequent reads.
+            def record_write bytes, state=nil
+                begin
+                    do_record_write bytes, state
+                rescue
+                    nil
+                end
+            end
+
+            # Returns next complete message from byte stream or nil.
+            def read_message bytes, state=nil
+                read_message! bytes.dup, state
+            end
+
+            # Mutating form of read_message. Removes a complete message
+            # from input stream. Returns the message or nil if there.
+            # is not a complete message.
+            def read_message! bytes, state=nil
+                begin
+                    do_read_message! bytes, state
+                rescue
+                    nil
+                end
+            end
+
+        end
+    end
+end
+
+require 'mu/pcap/reader/http_family'

data/lib/woolen_common/pcap/mu/pcap/reader/http_family.rb

@@ -0,0 +1,175 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'mu/pcap/reader'
+require 'stringio'
+require 'zlib'
+
+module Mu
+    class Pcap
+        class Reader
+
+# Reader for HTTP family of protocols (HTTP/SIP/RTSP).
+# Handles message boundaries and decompressing/dechunking payloads.
+            class HttpFamily < Reader
+                FAMILY = :http
+                FAMILY_TO_READER[FAMILY] = self
+                CRLF = "\r\n"
+
+                def family
+                    FAMILY
+                end
+
+                def do_record_write bytes, state=nil
+                    return if not state
+                    if bytes =~ RE_REQUEST_LINE
+                        method = $1
+                        requests = state[:requests] ||= []
+                        requests << method
+                    end
+                end
+
+                private :do_record_write
+
+                RE_CONTENT_ENCODING = /^content-encoding:\s*(gzip|deflate)/i
+                RE_CHUNKED = /Transfer-Encoding:\s*chunked/i
+                RE_HEADERS_COMPLETE = /.*?\r\n\r\n/m
+                # Request line e.g. GET /index.html HTTP/1.1
+                RE_REQUEST_LINE = /\A([^ \t\r\n]+)[ \t]+([^ \t\r\n]+)[ \t]+(HTTP|SIP|RTSP)\/[\d.]+.*\r\n/
+                # Status line e.g. SIP/2.0 404 Authorization required
+                RE_STATUS_LINE = /\A((HTTP|SIP|RTSP)\/[\d.]+[ \t]+(\d+))\b.*\r\n/
+
+                RE_CONTENT_LENGTH = /^(Content-Length)(:\s*)(\d+)\r\n/i
+                RE_CONTENT_LENGTH_SIP = /^(Content-Length|l)(:\s*)(\d+)\r\n/i
+
+
+                def do_read_message! bytes, state=nil
+                    case bytes
+                        when RE_REQUEST_LINE
+                            proto = $3
+                        when RE_STATUS_LINE
+                            proto = $2
+                            status = $3.to_i
+                            if state
+                                requests = state[:requests] ||= []
+                                if requests[0] == "HEAD"
+                                    reply_to_head = true
+                                end
+                                if status > 199
+                                    # We have a final response. Forget about request.
+                                    requests.shift
+                                end
+                            end
+                        else
+                            return nil # Not http family.
+                    end
+
+                    # Read headers
+                    if bytes =~ RE_HEADERS_COMPLETE
+                        headers = $&
+                        rest = $'
+                    else
+                        return nil
+                    end
+                    message = [headers]
+
+                    # Read payload.
+                    if proto == 'SIP'
+                        re_content_length = RE_CONTENT_LENGTH_SIP
+                    else
+                        re_content_length = RE_CONTENT_LENGTH
+                    end
+                    if reply_to_head
+                        length = 0
+                    elsif headers =~ RE_CHUNKED
+                        # Read chunks, dechunking in runtime case.
+                        raw, dechunked = get_chunks(rest)
+                        if raw
+                            length = raw.length
+                            payload = raw
+                        else
+                            return nil # Last chunk not received.
+                        end
+                    elsif headers =~ re_content_length
+                        length = $3.to_i
+                        if rest.length >= length
+                            payload = rest.slice(0, length)
+                        else
+                            return nil # Not enough bytes.
+                        end
+                    else
+                        # XXX. When there is a payload and no content-length
+                        # header HTTP RFC says to read until connection close.
+                        length = 0
+                    end
+
+                    message << payload
+
+                    # Consume message from input bytes.
+                    message_len = headers.length + length
+                    if bytes.length >= message_len
+                        bytes.slice!(0, message_len)
+                        return message.join
+                    else
+                        return nil # Not enough bytes.
+                    end
+                end
+
+                private :do_read_message!
+
+                # Returns array containing raw and dechunked payload. Returns nil
+                # if payload cannot be completely read.
+                RE_CHUNK_SIZE_LINE = /\A([[:xdigit:]]+)\r\n?/
+
+                def get_chunks bytes
+                    raw = []
+                    dechunked = []
+                    io = StringIO.new bytes
+                    until io.eof?
+                        # Read size line
+                        size_line = io.readline
+                        raw << size_line
+                        if size_line =~ RE_CHUNK_SIZE_LINE
+                            chunk_size = $1.to_i(16)
+                        else
+                            # Malformed chunk size line
+                            $stderr.puts "malformed size line : #{size_line.inspect}"
+                            return nil
+                        end
+
+                        # Read chunk data
+                        chunk = io.read(chunk_size)
+                        if chunk.size < chunk_size
+                            # malformed/incomplete
+                            $stderr.puts "malformed/incomplete #{chunk_size}"
+                            return nil
+                        end
+                        raw << chunk
+                        dechunked << chunk
+                        # Get end-of-chunk CRLF
+                        crlf = io.read(2)
+                        if crlf == CRLF
+                            raw << crlf
+                        else
+                            # CRLF has not arrived or, if this is the last chunk,
+                            # we might be looking at the first two bytes of a trailer
+                            # and we don't support trailers (see rfc 2616 sec3.6.1).
+                            return nil
+                        end
+
+                        if chunk_size == 0
+                            # Done. Return raw and dechunked payloads.
+                            return raw.join, dechunked.join
+                        end
+                    end
+
+                    # EOF w/out reaching last chunk.
+                    return nil
+                end
+            end
+
+        end
+    end
+end

data/lib/woolen_common/pcap/mu/pcap/sctp.rb

@@ -0,0 +1,369 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+    class Pcap
+
+        class SCTP < Packet
+            attr_accessor :src_port, :dst_port, :verify_tag, :checksum
+
+            # SCTP chunk types
+            CHUNK_DATA = 0x00
+            CHUNK_INIT = 0x01
+            CHUNK_INIT_ACK = 0x02
+            CHUNK_SACK = 0x03
+            CHUNK_HEARTBEAT = 0x04
+            CHUNK_HEARTBEAT_ACK = 0x05
+            CHUNK_ABORT = 0x06
+            CHUNK_SHUTDOWN = 0x07
+            CHUNK_SHUTDOWN_ACK = 0x08
+            CHUNK_ERROR = 0x09
+            CHUNK_COOKIE_ECHO = 0x0A
+            CHUNK_COOKIE_ACK = 0x0B
+            CHUNK_ECNE = 0x0C
+            CHUNK_CWR = 0x0D
+            CHUNK_SHUTDOWN_COMPLETE = 0x0E
+            CHUNK_AUTH = 0x0F
+            CHUNK_ASCONF_ACK = 0x80
+            CHUNK_PADDING = 0x84
+            CHUNK_FORWARD_TSN = 0xC0
+            CHUNK_ASCONF = 0xC1
+
+            # SCTP parameter types
+            PARAM_IPV4 = 0x0005
+            PARAM_IPV6 = 0x0006
+            PARAM_STATE_COOKIE = 0x0007
+            PARAM_COOKIE_PRESERVATIVE = 0x0009
+            PARAM_HOST_NAME_ADDR = 0x000B
+            PARAM_SUPPORTED_ADDR_TYPES = 0x000C
+            PARAM_ECN = 0x8000
+            PARAM_RANDOM = 0x8002
+            PARAM_CHUNK_LIST = 0x8003
+            PARAM_HMAC_ALGORITHM = 0x8004
+            PARAM_PADDING = 0x8005
+            PARAM_SUPPORTED_EXTENSIONS = 0x8006
+            PARAM_FORWARD_TSN = 0xC000
+            PARAM_SET_PRIMARY_ADDR = 0xC004
+            PARAM_ADAPTATION_LAYER_INDICATION = 0xC006
+
+            def initialize
+                super
+
+                @src_port = 0
+                @dst_port = 0
+                @verify_tag = 0
+                @checksum = 0
+                @payload = []
+            end
+
+            def flow_id
+                return [:sctp, @src_port, @dst_port, @verify_tag]
+            end
+
+            def reverse_flow_id
+                return [:sctp, @dst_port, @src_port, @checksum]
+            end
+
+            # Creates SCTP packet from the payload
+            def self.from_bytes bytes
+                # Basic packet validation
+                Pcap.assert(bytes.length >= 12,
+                            "Truncated SCTP header: 12 > #{bytes.length}")
+                Pcap.assert(bytes.length >= 16,
+                            "Truncated SCTP packet: got only #{bytes.length} bytes")
+
+                # Read SCTP header
+                sport, dport, vtag, cksum = bytes.unpack('nnNN')
+
+                # Create SCTP packet and populate SCTP header fields
+                sctp = SCTP.new
+                sctp.src_port = sport
+                sctp.dst_port = dport
+                sctp.verify_tag = vtag
+                sctp.checksum = cksum
+
+                # Initialize the counter
+                length = 12
+
+                # Collect the chunks
+                while length < bytes.length
+                    # Parse new chunk from the bytes
+                    chunk = Chunk.from_bytes(bytes[length..-1])
+
+                    # Get chunk size with padding
+                    length += chunk.padded_size
+
+                    # Add chunk to the list
+                    sctp << chunk
+                end
+
+                # Sync the payload
+                sctp.sync_payload
+
+                # Return the result
+                return sctp
+            end
+
+            class ReorderError < StandardError;
+            end
+
+            # Reorders SCTP packets, if necessary
+            def self.reorder packets
+                # Initialize
+                tsns = {}
+                init_packets = {}
+                init_ack_packets = {}
+                reordered_packets = []
+
+                # Iterate over each packet
+                while not packets.empty?
+                    # Get next packet
+                    packet = packets.shift
+
+                    # Do not reorder non-SCTP packets
+                    if not sctp?(packet)
+                        reordered_packets << packet
+                    else
+                        # Get SCTP portion
+                        sctp = packet.payload.payload
+
+                        # Sanity checks and packet filtering/preprocessing
+                        if 0 == sctp.verify_tag and not sctp.init?
+                            # Non-Init packet with 0 verify tag
+                            raise ReorderError, "Non-Init packet with zero verify tag"
+                        elsif sctp.init_or_ack? and 1 < sctp.chunk_count
+                            # Init/InitAck packet with more with one chunk
+                            raise ReorderError, "Init/Ack packet with more than 1 chunk"
+                        elsif sctp.init?
+                            # Use checksum to save reverse verify tag in the Init packet
+                            sctp.checksum = sctp[0].init_tag
+
+                            # Save orphaned Init packets until we find the Ack
+                            init_packets[sctp.reverse_flow_id] = sctp
+
+                            # Add packet for further processing
+                            reordered_packets << packet
+                        elsif sctp.init_ack?
+                            # Lookup Init packet and construct it's flow it
+                            init_packet = init_packets.delete(sctp.flow_id)
+
+                            # Did we find anything?
+                            if init_packet
+                                # Set verify tag in the Init packet
+                                init_packet.verify_tag = sctp[0].init_tag
+
+                                # Set reverse verify tag in the InitAck packet
+                                sctp.checksum = init_packet.verify_tag
+
+                                # Re-insert INIT packet keyed by its flow id
+                                init_packets[init_packet.flow_id] = init_packet
+                            else
+                                Pcap.warning("Orphaned SCTP INIT_ACK packet")
+                            end
+
+                            # Save InitAck packet
+                            init_ack_packets[sctp.flow_id] = sctp
+
+                            # Add packet for further processing
+                            reordered_packets << packet
+                        elsif sctp.has_data?
+                            # SCTP packet with user data; lookup Init or InitAck packet
+                            init_packet = init_packets[sctp.flow_id]
+                            init_ack_packet = init_ack_packets[sctp.flow_id]
+
+                            # It should belong to either one flow id or the other
+                            if init_packet
+                                # Set reverse verify tag from Init packet
+                                sctp.checksum = init_packet.checksum
+                            elsif init_ack_packet
+                                # Set reverse flow id from InitAck packet
+                                sctp.checksum = init_ack_packet.checksum
+                            else
+                                # Orphaned SCTP packet -- not very good
+                                Pcap.warning("Orphaned SCTP DATA packet detected")
+                            end
+
+                            # If we have just one chunk we are done
+                            if 1 == sctp.chunk_count and not tsns.member?(sctp[0].tsn)
+                                # Save TSN
+                                tsns[sctp[0].tsn] = sctp[0]
+
+                                # sync the payload
+                                sctp.sync_payload
+
+                                # Add packet for further processing
+                                reordered_packets << packet
+                            else
+                                # Split each data chunk in a separate SCTP packet
+                                sctp.chunk_count.times do
+                                    # Get next chunk
+                                    chunk = sctp.shift
+
+                                    # Is it data?
+                                    if CHUNK_DATA == chunk.type
+                                        # Yes, check for duplicate TSNs
+                                        if not tsns.member?(chunk.tsn)
+                                            # Not a duplicate; create new SCTP packet
+                                            packet_new = packet.deepdup
+
+                                            # Create new SCTP payload
+                                            sctp_new = SCTP.new
+                                            sctp_new.src_port = sctp.src_port
+                                            sctp_new.dst_port = sctp.dst_port
+                                            sctp_new.verify_tag = sctp.verify_tag
+                                            sctp_new.checksum = sctp.checksum
+
+                                            # Add the chunk
+                                            sctp_new << chunk
+
+                                            # Add SCTP payload to the new packet
+                                            packet_new.payload.payload = sctp_new
+
+                                            # Save TSN
+                                            tsns[chunk.tsn] = chunk
+
+                                            # Sync the payload
+                                            sctp_new.sync_payload
+
+                                            # Add packet for further processing
+                                            reordered_packets << packet_new
+                                        else
+                                            Pcap.warning("Duplicate chunk: #{chunk.tsn}")
+                                        end
+                                    else
+                                        Pcap.warning("Non-data chunk: #{chunk.type}")
+                                    end
+                                end
+                            end
+                        else
+                            # Other SCTP packet; we are not interested at this time
+                        end
+                    end
+                end
+
+                # Return the result
+                return reordered_packets
+            end
+
+            def write io, ip
+                # Give a warning if packet size exceeds maximum allowed
+                if @payload_raw and @payload_raw.length + 20 > 65535
+                    Pcap.warning("SCTP payload is too large")
+                end
+
+                # Calculate CRC32 checksum on the packet; temporarily removed due to a
+                # hack that uses checksum to link forward and reverse SCTP flow IDs.
+                #header = [@src_port, @dst_port, @verify_tag, 0].pack('nnNN')
+                #checksum = SCTP.crc32(header + @payload_raw)
+                header = [@src_port, @dst_port, @verify_tag, @checksum].pack('nnNN')
+
+                # Write SCTP header followed by each chunk
+                io.write(header)
+
+                # Write each chunks' data
+                @payload.each do |chunk|
+                    chunk.write(io, ip)
+                end
+            end
+
+            def sync_payload
+                # Reset raw bytes
+                @payload_raw = ''
+
+                # Iterate over each chunk
+                @payload.each do |chunk|
+                    @payload_raw << chunk.payload_raw
+                end
+
+                # Reset raw payload if it's empty
+                @payload_raw = nil if @payload_raw == ''
+            end
+
+            def self.crc32 bytes
+                r = 0xFFFFFFFF
+
+                bytes.each_byte do |b|
+                    r ^= b
+
+                    8.times do
+                        r = (r >> 1) ^ (0xEDB88320 * (r & 1))
+                    end
+                end
+
+                return r ^ 0xFFFFFFFF
+            end
+
+            def self.sctp? packet
+                return packet.is_a?(Ethernet) &&
+                    packet.payload.is_a?(IP) &&
+                    packet.payload.payload.is_a?(SCTP)
+            end
+
+            def << chunk
+                @payload << chunk
+            end
+
+            def shift
+                return @payload.shift
+            end
+
+            def [] index
+                return @payload[index]
+            end
+
+            def chunk_count
+                return @payload.size
+            end
+
+            def has_data?
+                return @payload.any? do |chunk|
+                    CHUNK_DATA == chunk.type
+                end
+            end
+
+            def to_s
+                return "sctp(%d, %d, %d, %s)" % [@src_port,
+                                                 @dst_port,
+                                                 @verify_tag,
+                                                 @payload.join(", ")]
+            end
+
+            def == other
+                return super &&
+                    self.src_port == other.src_port &&
+                    self.dst_port == other.dst_port &&
+                    self.verify_tag == other.verify_tag &&
+                    self.payload.size == other.payload.size
+            end
+
+            def init?
+                if CHUNK_INIT == @payload[0].type
+                    return true
+                else
+                    return false
+                end
+            end
+
+            def init_ack?
+                if CHUNK_INIT_ACK == @payload[0].type
+                    return true
+                else
+                    return false
+                end
+            end
+
+            def init_or_ack?
+                if CHUNK_INIT == @payload[0].type or CHUNK_INIT_ACK == @payload[0].type
+                    return true
+                else
+                    return false
+                end
+            end
+        end # class SCTP
+
+    end # class Pcap
+end # module Mu
+
+require 'mu/pcap/sctp/chunk'