pcapr-local 0.1.10

data/lib/mu/pcap/ethernet.rb

@@ -0,0 +1,148 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+class Ethernet < Packet
+    attr_accessor :src, :dst, :type
+    
+    ETHERTYPE_IP  = 0x0800
+    ETHERTYPE_IP6 = 0x86dd
+    ETHERTYPE_ARP = 0x0806
+    ETHERTYPE_PPPOE_SESSION = 0x8864
+    ETHERTYPE_802_1Q = 0X8100
+
+    PPP_IP   = 0x0021
+    PPP_IPV6 = 0x0057 
+
+    def initialize src=nil, dst=nil, type=0
+        super()
+        @src = src
+        @dst = dst
+        @type = type
+    end
+
+    def flow_id
+        if not @payload or @payload.is_a? String
+            return [:ethernet, @src, @dst, @type]
+        else
+            return @payload.flow_id
+        end
+    end
+
+    FMT_MAC = "C6"
+    FMT_n = 'n'
+    MAC_TEMPLATE = '%02x:%02x:%02x:%02x:%02x:%02x'
+    def self.from_bytes bytes
+        if bytes.length < 14
+            raise ParseError, "Truncated Ethernet header: expected 14 bytes, got #{bytes.length} bytes"
+        end
+
+        dst = bytes.slice!(0,6).unpack FMT_MAC
+        dst = MAC_TEMPLATE % dst
+        src = bytes.slice!(0,6).unpack FMT_MAC
+        src = MAC_TEMPLATE % src
+        type = bytes.slice!(0,2).unpack(FMT_n)[0]
+        while (type == ETHERTYPE_802_1Q)
+            # Skip 4 bytes for 802.1q vlan tag field
+            bytes.slice!(0,2)
+            type = bytes.slice!(0,2).unpack(FMT_n)[0]
+        end
+        ethernet = Ethernet.new src, dst, type
+        ethernet.payload = bytes
+        ethernet.payload_raw = bytes
+        begin
+            case type
+            when ETHERTYPE_IP
+                ethernet.payload = IPv4.from_bytes bytes
+            when ETHERTYPE_IP6
+                ethernet.payload = IPv6.from_bytes bytes
+            when ETHERTYPE_PPPOE_SESSION
+                # Remove PPPoE/PPP session layer
+                ethernet.payload = bytes
+                ethernet.remove_pppoe!
+            else
+                ethernet.payload = bytes
+            end
+        rescue ParseError => e
+            Pcap.warning e
+        end
+        return ethernet
+    end
+
+    def ip?
+        return payload.is_a?(IP)
+    end
+    
+    ADDR_TO_BYTES = {}
+    FMT_HEADER = 'a6a6n'
+    def write io
+        dst_mac = ADDR_TO_BYTES[@dst] ||= @dst.split(':').inject('') {|m, b| m << b.to_i(16).chr}
+        src_mac = ADDR_TO_BYTES[@src] ||= @src.split(':').inject('') {|m, b| m << b.to_i(16).chr}
+        bytes = [dst_mac, src_mac, @type].pack(FMT_HEADER)
+        io.write bytes
+        if @payload.is_a? String
+            io.write @payload
+        else
+            @payload.write io
+        end
+    end
+
+    # Remove the PPPoE and PPP headers.  PPPoE is documented in RFC 2516.
+    def remove_pppoe!
+        bytes = self.payload_raw
+
+        # Remove PPPoE header
+        Pcap.assert bytes.length >= 6, 'Truncated PPPoE header: ' +
+            "expected at least 6 bytes, got #{bytes.length} bytes"
+        version_type, code, session_id, length = bytes.unpack 'CCnn'
+        version = version_type >> 4 & 0b1111
+        type    = version_type      & 0b1111
+        Pcap.assert version == 1, "Unknown PPPoE version: #{version}"
+        Pcap.assert type == 1, "Unknown PPPoE type: #{type}"
+        Pcap.assert code == 0, "Unknown PPPoE code: #{code}"
+        bytes = bytes[6..-1]
+        Pcap.assert bytes.length >= length, 'Truncated PPoE packet: ' +
+            "expected #{length} bytes, got #{bytes.length} bytes"
+        
+        # Remove PPP header
+        Pcap.assert bytes.length >= 2, 'Truncated PPP packet: ' +
+            "expected at least bytes, got #{bytes.length} bytes"
+        protocol_id, = bytes.unpack 'n'
+        bytes = bytes[2..-1]
+        case protocol_id
+        when PPP_IP
+            self.payload = IPv4.from_bytes bytes
+            self.payload_raw = bytes
+            self.type = ETHERTYPE_IP
+        when PPP_IPV6
+            self.payload = IPv6.from_bytes bytes
+            self.payload_raw = bytes
+            self.type = ETHERTYPE_IP6
+        else
+            # Failed.  Don't update payload or type.
+            raise ParseError, "Unknown PPP protocol: 0x#{'%04x' % protocol_id}"
+        end
+    end
+
+    def to_s
+        if @payload.is_a? String
+            payload = @payload.inspect
+        else
+            payload = @payload.to_s
+        end
+        return "ethernet(%s, %s, %d, %s)" % [@src, @dst, @type, payload]
+    end
+
+    def == other
+        return super &&
+            self.src  == other.src &&
+            self.dst  == other.dst &&
+            self.type == other.type
+    end
+end
+
+end
+end

data/lib/mu/pcap/header.rb

@@ -0,0 +1,75 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+class Header
+    attr_accessor :magic, :version_major, :version_minor, :thiszone, :sigfigs,
+        :snaplen, :linktype
+
+    BIG_ENDIAN_FORMAT = 'nnNNNN'
+    LITTLE_ENDIAN_FORMAT = 'vvVVVV'
+
+    UNSUPPORTED_FORMATS =  { 
+        0x474D4255 => "NetMon", # "GMBU"
+        0x5452534E => "NA Sniffer (DOS)" # Starts with "TRSNIFF data"
+    }
+
+    def initialize
+        @magic = BIG_ENDIAN
+        @version_major = 2
+        @version_minor = 4
+        @thiszone = 0
+        @sigfigs = 0
+        @snaplen = 1500
+        @linktype = DLT_NULL
+    end
+
+    def self.read ios
+        header = Header.new
+        bytes = ios.read 24
+        Pcap.assert bytes, 'PCAP header missing'
+        Pcap.assert bytes.length == 24, 'Truncated PCAP header: ' +
+            "expected 24 bytes, got #{bytes.length} bytes"
+        header.magic, _ = bytes[0, 4].unpack 'N'
+        if header.magic == BIG_ENDIAN
+            format = BIG_ENDIAN_FORMAT
+        elsif header.magic == LITTLE_ENDIAN
+            format = LITTLE_ENDIAN_FORMAT
+        else 
+            format = UNSUPPORTED_FORMATS[header.magic]
+            if format.nil?
+                err = "Unsupported packet capture format. "
+            else
+                err = "#{format} capture files are not supported. "
+            end
+            raise ParseError, err
+        end
+        header.version_major, header.version_minor, header.thiszone,
+            header.sigfigs, header.snaplen, header.linktype = 
+            bytes[4..-1].unpack format
+        return header
+    end
+
+    def write io
+        bytes = [BIG_ENDIAN, @version_major, @version_minor, @thiszone,
+                 @sigfigs, @snaplen, DLT_EN10MB].pack('N' + BIG_ENDIAN_FORMAT)
+        io.write bytes
+    end
+
+    def == other
+        return self.class      == other.class &&
+            self.magic         == other.magic &&
+            self.version_major == other.version_major &&
+            self.version_minor == other.version_minor &&
+            self.thiszone      == other.thiszone &&
+            self.sigfigs       == other.sigfigs &&
+            self.snaplen       == other.snaplen &&
+            self.linktype      == other.linktype
+    end
+end
+
+end
+end

data/lib/mu/pcap/io_pair.rb

@@ -0,0 +1,67 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+# For emulating of a pair of connected sockets. Bytes written 
+# with #write to one side are returned by a subsequent #read on 
+# the other side.
+#
+# Use Pair.stream_pair to get a pair with stream semantics.
+# Use Pair.packet_pair to get a pair with packet semantics.
+class IOPair
+    attr_reader :read_queue
+    attr_accessor :other
+
+    def initialize
+        raise NotImplementedError
+    end
+
+    def self.stream_pair
+        io1 = Stream.new
+        io2 = Stream.new
+        io1.other = io2
+        io2.other = io1
+        return io1, io2
+    end
+
+    def self.packet_pair
+        io1 = Packet.new
+        io2 = Packet.new
+        io1.other = io2
+        io2.other = io1
+        return io1, io2
+    end
+
+    def write bytes
+        @other.read_queue << bytes
+        bytes.size
+    end
+
+    class Stream < IOPair
+        def initialize 
+            @read_queue = ""
+        end
+
+        def read n=nil
+            n ||= @read_queue.size
+            @read_queue.slice!(0,n)
+        end
+    end
+
+    class Packet < IOPair
+        def initialize 
+            @read_queue = []
+        end
+
+        def read 
+            @read_queue.shift
+        end
+    end
+
+end
+end
+end
+

data/lib/mu/pcap/io_wrapper.rb

@@ -0,0 +1,76 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'mu/pcap/io_pair'
+
+module Mu
+class Pcap
+class IOWrapper
+    attr_reader :ios, :unread, :state
+
+    def initialize ios, reader
+        @ios       = ios
+        @reader    = reader
+        # parse state for reader
+        @state     = {}
+        # read off of underlying io but not yet processed by @reader
+        @unread    = "" 
+    end
+
+    # Impose upper limit to protect against memory exhaustion.
+    MAX_RECEIVE_SIZE = 1048576 # 1MB
+
+    # Returns next higher level protocol message.
+    def read
+        until message = @reader.read_message!(@unread, @state)
+            bytes = @ios.read
+            if bytes and not bytes.empty?
+                @unread << bytes
+            else
+                return nil 
+            end
+            if @unread.size > MAX_RECEIVE_SIZE 
+                raise "Maximum message size (#{MAX_RECEIVE_SIZE}) exceeded"
+            end
+        end
+
+        return message
+    end
+
+    # Parser may need to see requests to understand responses.
+    def record_write bytes
+        @reader.record_write bytes, @state
+    end
+
+    def write bytes, *args
+        w = @ios.write bytes, *args
+        record_write bytes
+        w
+    end
+
+    def write_to bytes, *args
+        w = @ios.write_to bytes, *args
+        record_write bytes
+        w
+    end
+
+    def open  
+        if block_given?
+            @ios.open { yield }
+        else
+            @ios.open
+        end
+    end
+
+    def open?
+        @ios.open?
+    end
+
+    def close
+        @ios.close
+    end
+    
+end
+end
+end

data/lib/mu/pcap/ip.rb

@@ -0,0 +1,61 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+class IP < Packet
+    IPPROTO_TCP      = 6
+    IPPROTO_UDP      = 17
+    IPPROTO_HOPOPTS  = 0
+    IPPROTO_ROUTING  = 43
+    IPPROTO_FRAGMENT = 44
+    IPPROTO_AH       = 51
+    IPPROTO_NONE     = 59
+    IPPROTO_DSTOPTS  = 60
+    IPPROTO_SCTP     = 132
+
+    attr_accessor :src, :dst
+
+    def initialize src=nil, dst=nil
+        super()
+        @src = src
+        @dst = dst
+    end
+
+    def v4?
+        return false
+    end
+
+    def v6?
+        return false
+    end
+
+    def proto
+        raise NotImplementedError
+    end
+
+    def pseudo_header payload_length
+        raise NotImplementedError
+    end
+
+    def == other
+        return super &&
+            self.src    == other.src &&
+            self.dst    == other.dst
+    end
+
+    def self.checksum bytes
+        if bytes.size & 1 == 1
+            bytes = bytes + "\0"
+        end 
+        sum = 0
+        bytes.unpack("n*").each {|n| sum += n }
+        sum = (sum & 0xffff) + (sum >> 16 & 0xffff)
+        ~sum & 0xffff
+    end
+end
+
+end
+end

data/lib/mu/pcap/ipv4.rb

@@ -0,0 +1,257 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'ipaddr'
+
+module Mu
+class Pcap
+
+class IPv4 < IP
+    IP_RF = 0x8000 # Reserved
+    IP_DF = 0x4000 # Don't fragment
+    IP_MF = 0x2000 # More fragments
+    IP_OFFMASK = 0x1fff
+
+    FMT_HEADER = 'CCnnnCCna4a4'
+
+    attr_accessor :ip_id, :offset, :ttl, :proto, :src, :dst, :dscp
+
+    def initialize src=nil, dst=nil, ip_id=0, offset=0, ttl=64, proto=0, dscp=0
+        super()
+        @ip_id  = ip_id
+        @offset = offset
+        @ttl    = ttl
+        @proto  = proto
+        @src    = src
+        @dst    = dst
+        @dscp   = dscp
+    end
+
+    def v4?
+        return true
+    end
+
+    def flow_id
+        if not @payload or @payload.is_a? String
+            return [:ipv4, @proto, @src, @dst]
+        else
+            return [:ipv4, @src, @dst, @payload.flow_id]
+        end
+    end
+
+    NTOP = {} # Network to human cache
+    HTON = {} # Human to network cache
+
+    def self.from_bytes bytes
+        bytes.length >= 20 or
+            raise ParseError, "Truncated IPv4 header: expected at least 20 bytes, got #{bytes.length} bytes"
+
+        vhl, tos, length, id, offset, ttl, proto, checksum, src, dst = bytes[0,20].unpack FMT_HEADER
+        version = vhl >> 4 
+        hl = (vhl & 0b1111) * 4
+
+        version == 4 or
+            raise ParseError, "Wrong IPv4 version: got (#{version})"
+        hl >= 20 or
+            raise ParseError, "Bad IPv4 header length: expected at least 20 bytes raise ParseError, got #{hl} bytes"
+        bytes.length >= hl or
+            raise ParseError, "Truncated IPv4 header: expected #{hl} bytes raise ParseError, got #{bytes.length} bytes"
+        length >= 20 or
+            raise ParseError, "Bad IPv4 packet length: expected at least 20 bytes raise ParseError, got #{length} bytes"
+        bytes.length >= length or
+            raise ParseError, "Truncated IPv4 packet: expected #{length} bytes raise ParseError, got #{bytes.length} bytes"
+
+        if hl != 20
+            IPv4.check_options bytes[20, hl-20]
+        end
+
+        src = NTOP[src] ||= IPAddr.ntop(src)
+        dst = NTOP[dst] ||= IPAddr.ntop(dst)
+        dscp = tos >> 2
+        ipv4 = IPv4.new(src, dst, id, offset, ttl, proto, dscp)
+        ipv4.payload_raw = bytes[hl..-1]
+
+        payload = bytes[hl...length]
+        if offset & (IP_OFFMASK | IP_MF) == 0
+            begin
+                case proto
+                when IPPROTO_TCP
+                    ipv4.payload = TCP.from_bytes payload
+                when IPPROTO_UDP
+                    ipv4.payload = UDP.from_bytes payload
+                when IPPROTO_SCTP
+                    ipv4.payload = SCTP.from_bytes payload
+                else
+                    ipv4.payload = payload
+                end
+            rescue ParseError => e
+                Pcap.warning e
+            end
+        else
+            ipv4.payload = payload
+        end
+        return ipv4
+    end
+
+    def write io
+        if @payload.is_a? String
+            payload = @payload
+        else
+            string_io = StringIO.new
+            @payload.write string_io, self
+            payload = string_io.string
+        end
+        length = 20 + payload.length
+        if length > 65535
+            Pcap.warning "IPv4 payload is too large"
+        end
+
+        src = HTON[@src] ||= IPAddr.new(@src).hton
+        dst = HTON[@dst] ||= IPAddr.new(@dst).hton
+        fields = [0x45, @dscp << 2, length, @ip_id, @offset, @ttl, @proto, 0, src, dst] 
+        header = fields.pack(FMT_HEADER)
+        fields[7] = IP.checksum(header)
+        header = fields.pack(FMT_HEADER)
+        io.write header
+        io.write payload
+    end
+
+    FMT_PSEUDO_HEADER = 'a4a4CCn'
+    def pseudo_header payload_length
+        src = HTON[@src] ||= IPAddr.new(@src).hton
+        dst = HTON[@dst] ||= IPAddr.new(@dst).hton
+        return [src, dst, 0, @proto, payload_length].pack(FMT_PSEUDO_HEADER)
+    end
+
+    def fragment?
+        return (@offset & (IP_OFFMASK | IP_MF) != 0)
+    end
+
+    # Check that IP or TCP options are valid.  Do nothing if they are valid.
+    # Both IP and TCP options are 8-bit TLVs with an inclusive length.  Both
+    # have one byte options 0 and 1.
+    def self.check_options options, label='IPv4'
+        while not options.empty?
+            type = options.slice!(0, 1)[0].ord
+            if type == 0 or type == 1
+                next
+            end
+            Pcap.assert !options.empty?,
+                "#{label} option #{type} is missing the length field"
+            length = options.slice!(0, 1)[0].ord
+            Pcap.assert length >= 2,
+                "#{label} option #{type} has invalid length: #{length}"
+            Pcap.assert length - 2 <= options.length,
+                "#{label} option #{type} has truncated data"
+            options.slice! 0, length - 2
+        end
+    end
+
+    ReassembleState = ::Struct.new :packets, :bytes, :mf, :overlap
+
+    # Reassemble fragmented IPv4 packets
+    def self.reassemble packets
+        reassembled_packets = []
+        flow_id_to_state = {}
+        packets.each do |packet|
+            if not packet.is_a?(Ethernet) or not packet.payload.is_a?(IPv4)
+                # Ignore non-IPv4 packet
+            elsif not packet.payload.fragment?
+                # Ignore non-fragments
+            else
+                # Get reassembly state
+                ip = packet.payload
+                flow_id = [ip.ip_id, ip.proto, ip.src, ip.dst]
+                state = flow_id_to_state[flow_id]
+                if not state
+                    state = ReassembleState.new [], [], true, false
+                    flow_id_to_state[flow_id] = state
+                end
+                state.packets << packet
+
+                # Clear the more-fragments flag if no more fragments
+                if ip.offset & IP_MF == 0
+                    state.mf = false
+                end
+
+                # Add the bytes
+                start = (ip.offset & IP_OFFMASK) * 8
+                finish = start + ip.payload.length
+                state.bytes.fill nil, start, finish - start
+                start.upto(finish-1) do |i|
+                    if not state.bytes[i]
+                        byte = ip.payload[i - start].chr
+                        state.bytes[i] = byte
+                    elsif not state.overlap
+                        name = "%s:%s:%d" % [ip.src, ip.dst, ip.proto]
+                        Pcap.warning \
+                            "IPv4 flow #{name} contains overlapping fragements"
+                        state.overlap = true
+                    end
+                end
+
+                # We're done if we've received a fragment without the
+                # more-fragments flag and all the bytes in the buffer have been
+                # set.
+                if not state.mf and state.bytes.all?
+                    # Remove fragments from reassembled_packets
+                    state.packets.each do |packet|
+                        reassembled_packets.delete_if do |reassembled_packet|
+                            packet.object_id == reassembled_packet.object_id
+                        end
+                    end
+                    # Remove state
+                    flow_id_to_state.delete flow_id
+                    # Create new packet
+                    packet = state.packets[0].deepdup
+                    ipv4 = packet.payload
+                    ipv4.offset = 0
+                    ipv4.payload = state.bytes.join
+                    # Decode
+                    begin
+                        case ipv4.proto
+                        when IPPROTO_TCP
+                            ipv4.payload = TCP.from_bytes ipv4.payload
+                        when IPPROTO_UDP
+                            ipv4.payload = UDP.from_bytes ipv4.payload
+                        when IPPROTO_SCTP
+                            ipv4.payload = SCTP.from_bytes ipv4.payload
+                        end
+                    rescue ParseError => e
+                        Pcap.warning e
+                    end
+                end
+            end
+            reassembled_packets << packet
+        end
+        if !flow_id_to_state.empty?
+            Pcap.warning \
+                "#{flow_id_to_state.length} flow(s) have IPv4 fragments " \
+                "that can't be reassembled"
+        end
+
+        return reassembled_packets
+    end
+
+    def to_s
+        if @payload.is_a? String
+            payload = @payload.inspect
+        else
+            payload = @payload.to_s
+        end
+        return "ipv4(%d, %s, %s, %s)" % [@proto, @src, @dst, payload]
+    end
+
+    def == other
+        return super &&
+            self.proto  == other.proto &&
+            self.ip_id  == other.ip_id &&
+            self.offset == other.offset &&
+            self.ttl    == other.ttl &&
+            self.dscp   == other.dscp
+    end
+end
+
+end
+end