DIY-pcap 0.2.5 → 0.2.6

data/lib/diy/parser/mu/pcap/udp.rb

@@ -0,0 +1,69 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+class UDP < Packet
+    attr_accessor :src_port, :dst_port
+
+    def initialize src_port=0, dst_port=0
+        super()
+        @src_port = src_port
+        @dst_port = dst_port
+    end
+
+    def flow_id
+        return [:udp, @src_port, @dst_port]
+    end
+
+    FMT_nnnn = 'nnnn'
+    def self.from_bytes bytes
+        bytes_length = bytes.length
+        bytes_length >= 8 or
+            raise ParseError, "Truncated UDP header: expected 8 bytes, got #{bytes_length} bytes"
+        sport, dport, length, checksum = bytes.unpack(FMT_nnnn)
+        bytes_length >= length or 
+            raise ParseError, "Truncated UDP packet: expected #{length} bytes, got #{bytes_length} bytes"
+        udp = UDP.new sport, dport
+        udp.payload_raw = bytes[8..-1]
+        udp.payload = bytes[8..length]
+        return udp
+    end
+
+    def write io, ip
+        length = @payload.length
+        length_8 = length + 8
+        if length_8 > 65535
+            Pcap.warning "UDP payload is too large"
+        end
+        pseudo_header = ip.pseudo_header length_8
+        header = [@src_port, @dst_port, length_8, 0] \
+            .pack FMT_nnnn
+        checksum = IP.checksum(pseudo_header + header + @payload)
+        header = [@src_port, @dst_port, length_8, checksum] \
+            .pack FMT_nnnn
+        io.write header
+        io.write @payload
+    end
+
+    def self.udp? packet
+        return packet.is_a?(Ethernet) &&
+            packet.payload.is_a?(IP) &&
+            packet.payload.payload.is_a?(UDP)
+    end
+
+    def to_s
+        return "udp(%d, %d, %s)" % [@src_port, @dst_port, @payload.inspect]
+    end
+
+    def == other
+        return super &&
+            self.src_port == other.src_port &&
+            self.dst_port == other.dst_port
+    end
+end
+
+end
+end

data/lib/diy/parser/mu/scenario/pcap.rb

@@ -0,0 +1,172 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'tempfile'
+require 'fileutils'
+require 'mu/scenario/pcap/fields'
+require 'mu/pcap'
+require 'json'
+
+module Mu
+class Scenario
+
+module Pcap
+    TSHARK_READ_TIMEOUT     = 10.0 # seconds
+    TSHARK_LINES_PER_PACKET = 16384
+    TSHARK_OPTS = "-n -o tcp.desegment_tcp_streams:false"
+    TSHARK_OPTS_SUFFIX = TSHARK_OPTS
+    TSHARK_SIZE_OPTS = "-n -o 'column.format: cum_size, \"%B\"'"
+    TSHARK_PSML_OPTS = %Q{#{TSHARK_OPTS} -o 'column.format: "Protocol", "%p", "Info", "%i"'}
+
+    MAX_PCAP_SIZE = 102400 # 100KB
+    MAX_RAW_PCAP_SIZE_MB = 25
+    MAX_RAW_PCAP_SIZE = MAX_RAW_PCAP_SIZE_MB * 1024 * 1000
+    EXCLUDE_FROM_SIZE_CHECK = ['rtp'].freeze
+
+    class PcapTooLarge < StandardError; end
+    
+    def self.reset_options options
+        return unless options
+        tshark_opts = options << ' ' << TSHARK_OPTS_SUFFIX
+        remove_const(:TSHARK_OPTS) if const_defined?(:TSHARK_OPTS)
+        const_set(:TSHARK_OPTS, tshark_opts)
+    end
+
+    def self.validate_pcap_size(path)
+        tshark_filter = EXCLUDE_FROM_SIZE_CHECK.map{ |proto| "not #{proto}" }.join " and "
+        io = ::IO.popen "tshark #{TSHARK_SIZE_OPTS} -r #{path} -R '#{tshark_filter}' | tail -1"
+        if ::IO.select [ io ], nil, nil, TSHARK_READ_TIMEOUT
+            if io.eof?
+                size = 0
+            else
+                last_line = io.readline
+                size = last_line.to_i
+            end
+        end
+
+        if size.nil? or size == 0
+            size = File.size(path)
+        end
+
+        if size > MAX_PCAP_SIZE
+            raise PcapTooLarge, "Selected packets have a size of #{size} bytes which " +
+                "exceeds the #{MAX_PCAP_SIZE} byte maximum."
+        end
+
+        if size > MAX_RAW_PCAP_SIZE
+            raise PcapTooLarge, "Selected packets have a raw size of #{size} bytes which " +
+                "exceeds the #{MAX_RAW_PCAP_SIZE_MB}MB maximum."
+        end
+
+        return size
+    end
+
+    PAR_VERSION = 1
+    def self.export_to_par pcap_path, opts=nil
+        opts ||= {}
+
+        # Open pcap file
+        File.exist?(pcap_path) or raise "Cannot open file '#{pcap_path}'."
+        validate_pcap_size pcap_path
+        pcap = open pcap_path, 'rb'
+
+        # Get Mu::Pcap::Packets
+        packets = to_pcap_packets pcap, opts[:isolate_l7]
+
+        # Write normalized packets to tempfile
+        tmpdir = Dir.mktmpdir
+        norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+        pcap = Mu::Pcap.from_packets packets
+        pcap.write norm_pcap
+        norm_pcap.close
+
+        # Get wireshark dissected field values for all packets.
+        `tshark -T fields #{TSHARK_OPTS} #{Fields::TSHARK_OPTS} -Eseparator='\xff' -r #{norm_pcap.path} > #{tmpdir}/fields`
+        fields = open "#{tmpdir}/fields", 'rb'
+
+        # Get wireshark dissected field values for all packets.
+        fields_array = []
+        if fields
+            packets.each do |packet|
+                fields_array <<  Fields.next_from_io(fields)
+            end
+        end
+
+        # Protocol specific preprocessing, packets may be deleted.
+        Rtp.preprocess packets, fields_array
+
+        File.open "#{tmpdir}/packets.dump", 'wb' do |f|
+            Marshal.dump packets, f
+        end
+
+        # Create a second pcap with packets removed.
+        norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+        pcap = Mu::Pcap.from_packets packets
+        pcap.write norm_pcap
+        norm_pcap.close
+
+        # Dump PSML to file.
+        # (The no-op filter sometimes produces slighty more verbose descriptions.)
+        `tshark -T psml #{TSHARK_PSML_OPTS} -r #{norm_pcap.path} -R 'rtp or not rtp' > #{tmpdir}/psml`
+
+        # Dump PDML io file.
+        `tshark -T pdml #{TSHARK_OPTS} -r #{norm_pcap.path} > #{tmpdir}/pdml`
+        pdml = open "#{tmpdir}/pdml", 'rb'
+
+        # Create about
+        open "#{tmpdir}/about", 'w' do |about|
+            about.puts({:par_version => PAR_VERSION}.to_json)
+        end
+
+        # Create zip
+        Dir.chdir tmpdir do
+            system "zip -q dissected.zip about pdml psml fields packets.dump normalized.pcap"
+            return open("dissected.zip")
+        end
+    ensure
+        if tmpdir
+            FileUtils.rm_rf tmpdir
+        end
+    end
+
+    def self.to_pcap_packets io, isolate_l7=true
+        packets = []
+
+        # Read Pcap packets from Pcap
+        Mu::Pcap.each_pkthdr io do |pkthdr|
+            if pkthdr.len != pkthdr.caplen
+                raise Mu::Pcap::ParseError, "Error: Capture contains truncated packets. " +
+                    "Try recapturing with an increased snapshot length."
+            end
+            if not pkthdr.pkt.is_a? Mu::Pcap::Ethernet
+                warning 'Unable to parse packet, skipping.'
+            end
+            packets << pkthdr.pkt
+        end
+
+        if (packets.length == 0)
+            raise Mu::Pcap::ParseError, "No valid packets found!"
+        end
+
+        packets = Mu::Pcap::IPv4.reassemble packets
+
+        if isolate_l7
+            packets = Mu::Pcap::Packet.isolate_l7 packets
+        end
+
+        packets = Mu::Pcap::Packet.normalize packets
+        packets = Mu::Pcap::TCP.split packets
+
+        packets
+    end
+
+    def self.warning msg
+        $stderr.puts "WARNING: #{msg}"#, caller, $!
+    end
+end
+
+end
+end
+
+require 'mu/scenario/pcap/rtp'

data/lib/diy/parser/mu/scenario/pcap/fields.rb

@@ -0,0 +1,50 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'mu/scenario/pcap'
+
+module Mu
+class Scenario
+module Pcap
+
+class Fields
+    FIELDS = [
+        :rtp,
+        :"rtp.setup-frame"
+    ].freeze
+    FIELD_COUNT = FIELDS.length
+    SEPARATOR   = "\xff".freeze
+    TSHARK_OPTS = "-Eseparator='#{SEPARATOR}'" 
+    FIELDS.each do |field|
+        TSHARK_OPTS << " -e #{field}"
+    end
+    TSHARK_OPTS.freeze
+
+    def self.readline io
+        if ::IO.select [ io ], nil, nil, Pcap::TSHARK_READ_TIMEOUT
+            return io.readline.chomp
+        end 
+        
+        raise Errno::ETIMEDOUT, "read timed out"
+    end
+
+    def self.next_from_io io
+        if line = readline(io)
+            fields = line.split SEPARATOR, FIELD_COUNT
+            hash = {}
+            FIELDS.each do |key|
+                val = fields.shift
+                hash[key] = val.empty? ? nil : val
+            end
+            return hash
+        end
+    rescue Exception => e
+        Pcap.warning e.message
+        return nil
+    end
+
+end
+end
+end
+end

data/lib/diy/parser/mu/scenario/pcap/rtp.rb

@@ -0,0 +1,71 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Scenario
+module Pcap
+module Rtp
+
+    TRUNC_COUNT = 5
+
+    def self.preprocess packets, fields_per_packet
+        signaled_by = []
+        prev_signal_frame = {}
+        packets.each_with_index do |packet, idx|
+            fields = fields_per_packet[idx]
+            if fields and fields[:rtp]
+                flow_id = packet.flow_id
+                if frame = fields[:"rtp.setup-frame"]
+                    prev_signal_frame[flow_id] = frame
+                else
+                    if frame = prev_signal_frame[flow_id]
+                        fields[:"rtp.setup-frame"] = frame
+                    else
+                        packets[idx] = nil
+                        fields_per_packet[idx] = nil
+                        next
+                   end
+                end
+                sig_idx = frame.to_i 
+                signaled_by[idx] = sig_idx
+            end
+        end
+
+        flow_to_count = Hash.new 0
+        prev_setup_frame = {} 
+        keep_frames = []
+        packets.each_with_index do |packet, idx|
+            if setup_frame = signaled_by[idx]
+                flow = packet.flow_id
+                count = flow_to_count[flow]
+                if setup_frame != prev_setup_frame[flow]
+                    prev_setup_frame[flow] = setup_frame
+                    count = 1
+                else
+                    count += 1 
+                end
+                if count <= TRUNC_COUNT
+                    keep_frames << idx + 1
+                else
+                    packets[idx] = nil
+                    fields_per_packet[idx] = nil
+                end
+                flow_to_count[flow] = count
+            end
+        end
+
+        packets.reject! {|p| not p }
+        fields_per_packet.reject! {|p| not p }
+
+        filter = "not rtp"
+        keep_frames.each do |frame|
+            filter << " or frame.number == #{frame}"
+        end
+
+        return filter
+    end
+end
+end
+end
+end

data/lib/diy/parser/pcap.rb

@@ -0,0 +1,113 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+require 'socket'
+require 'stringio'
+require 'mu/fixnum_ext'
+
+module Mu
+
+class Pcap
+    class ParseError < StandardError ; end
+
+    LITTLE_ENDIAN = 0xd4c3b2a1
+    BIG_ENDIAN    = 0xa1b2c3d4
+
+    DLT_NULL      = 0
+    DLT_EN10MB    = 1
+    DLT_RAW       = 12 # DLT_LOOP in OpenBSD
+    DLT_LINUX_SLL = 113
+
+    attr_accessor :header, :pkthdrs
+    
+    def initialize
+        @header = Header.new
+        @pkthdrs = []
+    end
+
+    # Read PCAP file from IO and return Mu::Pcap.  If decode is true, also
+    # decode the Pkthdr packet contents to Mu::Pcap objects.
+    def self.read io, decode=true
+        pcap = Pcap.new
+        pcap.header = each_pkthdr(io, decode) do |pkthdr|
+            pcap.pkthdrs << pkthdr
+        end
+        return pcap
+    end
+
+    # Create PCAP from list of packets.
+    def self.from_packets packets
+        pcap = Pcap.new
+        packets.each do |packet|
+            pkthdr = Mu::Pcap::Pkthdr.new
+            pkthdr.pkt = packet
+            pcap.pkthdrs << pkthdr
+        end
+        return pcap
+    end
+
+    # Write PCAP file to IO.  Uses big-endian and linktype EN10MB.
+    def write io
+        @header.write io
+        @pkthdrs.each do |pkthdr|
+            pkthdr.write io
+        end
+    end
+
+    # Read PCAP packet headers from IO and return Mu::Pcap::Header.  If decode
+    # is true, also decode the Pkthdr packet contents to Mu::Pcap objects.  Use
+    # this for large files when each packet header can processed independently
+    # - it will perform better.
+    def self.each_pkthdr io, decode=true
+        header = Header.read io
+        while not io.eof?
+            pkthdr = Pkthdr.read io, header.magic
+            if decode
+                pkthdr.decode! header.magic, header.linktype
+            end
+            yield pkthdr
+        end
+        return header
+    end
+
+    # Read packets from PCAP
+    def self.read_packets io, decode=true
+        packets = []
+        each_pkthdr(io) { |pkthdr| packets << pkthdr.pkt }
+        return packets
+    end
+
+    # Assertion used during Pcap parsing
+    def self.assert cond, msg
+        if not cond
+            raise ParseError, msg
+        end
+    end
+
+    # Warnings from Pcap parsing are printed using this method.
+    def self.warning msg
+        $stderr.puts "WARNING: #{msg}"
+    end
+
+    def == other
+        return self.class == other.class &&
+            self.header   == other.header &&
+            self.pkthdrs  == other.pkthdrs
+    end
+end
+
+end
+
+require 'mu/pcap/header'
+require 'mu/pcap/pkthdr'
+require 'mu/pcap/packet'
+require 'mu/pcap/ethernet'
+require 'mu/pcap/ip'
+require 'mu/pcap/ipv4'
+require 'mu/pcap/ipv6'
+require 'mu/pcap/tcp'
+require 'mu/pcap/udp'
+require 'mu/pcap/sctp'