woolen_common 0.0.1

data/lib/woolen_common/pcap/mu/pcap/udp.rb

@@ -0,0 +1,81 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+    class Pcap
+
+        class UDP < Packet
+            class << self
+                def build_pkt(ip_pkt,src_port,dst_port)
+                    if ip_pkt && ip_pkt.kind_of?(IP)
+                        the_udp_pkt = self.new(src_port,dst_port)
+                        ip_pkt.payload = the_udp_pkt
+                    else
+                        raise 'can not build udp pkt with not ip pkt'
+                    end
+                end
+            end
+            attr_accessor :src_port, :dst_port
+
+            def initialize(src_port=0, dst_port=0)
+                super()
+                @src_port = src_port
+                @dst_port = dst_port
+            end
+
+            def flow_id
+                return [:udp, @src_port, @dst_port]
+            end
+
+            FMT_nnnn = 'nnnn'
+
+            def self.from_bytes bytes
+                bytes_length = bytes.length
+                bytes_length >= 8 or
+                    raise ParseError, "Truncated UDP header: expected 8 bytes, got #{bytes_length} bytes"
+                sport, dport, length, checksum = bytes.unpack(FMT_nnnn)
+                bytes_length >= length or
+                    raise ParseError, "Truncated UDP packet: expected #{length} bytes, got #{bytes_length} bytes"
+                udp = UDP.new sport, dport
+                udp.payload_raw = bytes[8..-1]
+                udp.payload = bytes[8..length]
+                return udp
+            end
+
+            def write io, ip
+                length = @payload.bytesize
+                length_8 = length + 8
+                if length_8 > 65535
+                    Pcap.warning "UDP payload is too large"
+                end
+                pseudo_header = ip.pseudo_header length_8
+                header = [@src_port, @dst_port, length_8, 0] \
+            .pack FMT_nnnn
+                checksum = IP.checksum(pseudo_header + header + @payload)
+                header = [@src_port, @dst_port, length_8, checksum] \
+            .pack FMT_nnnn
+                io.write header
+                io.write @payload
+            end
+
+            def self.udp? packet
+                return packet.is_a?(Ethernet) &&
+                    packet.payload.is_a?(IP) &&
+                    packet.payload.payload.is_a?(UDP)
+            end
+
+            def to_s
+                return "udp(%d, %d, %s)" % [@src_port, @dst_port, @payload.inspect]
+            end
+
+            def == other
+                return super &&
+                    self.src_port == other.src_port &&
+                    self.dst_port == other.dst_port
+            end
+        end
+
+    end
+end

data/lib/woolen_common/pcap/mu/scenario/pcap.rb

@@ -0,0 +1,175 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'tempfile'
+require 'fileutils'
+require 'mu/scenario/pcap/fields'
+require 'mu/pcap'
+require 'json'
+
+module Mu
+    class Scenario
+
+        module Pcap
+            TSHARK_READ_TIMEOUT = 10.0 # seconds
+            TSHARK_LINES_PER_PACKET = 16384
+            TSHARK_OPTS = "-n -o tcp.desegment_tcp_streams:false"
+            TSHARK_OPTS_SUFFIX = TSHARK_OPTS
+            TSHARK_SIZE_OPTS = "-n -o 'column.format: cum_size, \"%B\"'"
+            TSHARK_PSML_OPTS = %Q{#{TSHARK_OPTS} -o 'column.format: "Protocol", "%p", "Info", "%i"'}
+
+            MAX_PCAP_SIZE = 102400 # 100KB
+            MAX_RAW_PCAP_SIZE_MB = 25
+            MAX_RAW_PCAP_SIZE = MAX_RAW_PCAP_SIZE_MB * 1024 * 1000
+            EXCLUDE_FROM_SIZE_CHECK = ['rtp'].freeze
+
+            class PcapTooLarge < StandardError;
+            end
+
+            def self.reset_options options
+                return unless options
+                tshark_opts = options << ' ' << TSHARK_OPTS_SUFFIX
+                remove_const(:TSHARK_OPTS) if const_defined?(:TSHARK_OPTS)
+                const_set(:TSHARK_OPTS, tshark_opts)
+            end
+
+            def self.validate_pcap_size(path)
+                tshark_filter = EXCLUDE_FROM_SIZE_CHECK.map { |proto| "not #{proto}" }.join " and "
+                io = ::IO.popen "tshark #{TSHARK_SIZE_OPTS} -r #{path} -R '#{tshark_filter}' | tail -1"
+                if ::IO.select [io], nil, nil, TSHARK_READ_TIMEOUT
+                    if io.eof?
+                        size = 0
+                    else
+                        last_line = io.readline
+                        size = last_line.to_i
+                    end
+                end
+
+                if size.nil? or size == 0
+                    size = File.size(path)
+                end
+
+                if size > MAX_PCAP_SIZE
+                    raise PcapTooLarge, "Selected packets have a size of #{size} bytes which " +
+                        "exceeds the #{MAX_PCAP_SIZE} byte maximum."
+                end
+
+                if size > MAX_RAW_PCAP_SIZE
+                    raise PcapTooLarge, "Selected packets have a raw size of #{size} bytes which " +
+                        "exceeds the #{MAX_RAW_PCAP_SIZE_MB}MB maximum."
+                end
+
+                return size
+            end
+
+            PAR_VERSION = 1
+
+            def self.export_to_par pcap_path, opts=nil
+                opts ||= {}
+
+                # Open pcap file
+                File.exist?(pcap_path) or raise "Cannot open file '#{pcap_path}'."
+                validate_pcap_size pcap_path
+                pcap = open pcap_path, 'rb'
+
+                # Get Mu::Pcap::Packets
+                packets = to_pcap_packets pcap, opts[:isolate_l7]
+
+                # Write normalized packets to tempfile
+                tmpdir = Dir.mktmpdir
+                norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+                pcap = Mu::Pcap.from_packets packets
+                pcap.write norm_pcap
+                norm_pcap.close
+
+                # Get wireshark dissected field values for all packets.
+                `tshark -T fields #{TSHARK_OPTS} #{Fields::TSHARK_OPTS} -Eseparator='\xff' -r #{norm_pcap.path} > #{tmpdir}/fields`
+                fields = open "#{tmpdir}/fields", 'rb'
+
+                # Get wireshark dissected field values for all packets.
+                fields_array = []
+                if fields
+                    packets.each do |packet|
+                        fields_array << Fields.next_from_io(fields)
+                    end
+                end
+
+                # Protocol specific preprocessing, packets may be deleted.
+                Rtp.preprocess packets, fields_array
+
+                File.open "#{tmpdir}/packets.dump", 'wb' do |f|
+                    Marshal.dump packets, f
+                end
+
+                # Create a second pcap with packets removed.
+                norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+                pcap = Mu::Pcap.from_packets packets
+                pcap.write norm_pcap
+                norm_pcap.close
+
+                # Dump PSML to file.
+                # (The no-op filter sometimes produces slighty more verbose descriptions.)
+                `tshark -T psml #{TSHARK_PSML_OPTS} -r #{norm_pcap.path} -R 'rtp or not rtp' > #{tmpdir}/psml`
+
+                # Dump PDML io file.
+                `tshark -T pdml #{TSHARK_OPTS} -r #{norm_pcap.path} > #{tmpdir}/pdml`
+                pdml = open "#{tmpdir}/pdml", 'rb'
+
+                # Create about
+                open "#{tmpdir}/about", 'w' do |about|
+                    about.puts({ :par_version => PAR_VERSION }.to_json)
+                end
+
+                # Create zip
+                Dir.chdir tmpdir do
+                    system "zip -q dissected.zip about pdml psml fields packets.dump normalized.pcap"
+                    return open("dissected.zip")
+                end
+            ensure
+                if tmpdir
+                    FileUtils.rm_rf tmpdir
+                end
+            end
+
+            def self.to_pcap_packets io, isolate_l7=true
+                packets = []
+
+                # Read Pcap packets from Pcap
+                Mu::Pcap.each_pkthdr io do |pkthdr|
+                    if pkthdr.len != pkthdr.caplen
+                        raise Mu::Pcap::ParseError, "Error: Capture contains truncated packets. " +
+                            "Try recapturing with an increased snapshot length."
+                    end
+                    if not pkthdr.pkt.is_a? Mu::Pcap::Ethernet
+                        warning 'Unable to parse packet, skipping.'
+                    end
+                    packets << pkthdr.pkt
+                end
+
+                if (packets.length == 0)
+                    raise Mu::Pcap::ParseError, "No valid packets found!"
+                end
+
+                packets = Mu::Pcap::IPv4.reassemble packets
+
+                if isolate_l7
+                    packets = Mu::Pcap::Packet.isolate_l7 packets
+                end
+
+                packets = Mu::Pcap::Packet.normalize packets
+                packets = Mu::Pcap::TCP.split packets
+
+                packets
+            end
+
+            def self.warning msg
+                $stderr.puts "WARNING: #{msg}" #, caller, $!
+            end
+        end
+
+    end
+end
+
+require 'mu/scenario/pcap/rtp'

data/lib/woolen_common/pcap/mu/scenario/pcap/fields.rb

@@ -0,0 +1,51 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'mu/scenario/pcap'
+
+module Mu
+    class Scenario
+        module Pcap
+
+            class Fields
+                FIELDS = [
+                    :rtp,
+                    :"rtp.setup-frame"
+                ].freeze
+                FIELD_COUNT = FIELDS.length
+                SEPARATOR = "\xff".freeze
+                TSHARK_OPTS = "-Eseparator='#{SEPARATOR}'"
+                FIELDS.each do |field|
+                    TSHARK_OPTS << " -e #{field}"
+                end
+                TSHARK_OPTS.freeze
+
+                def self.readline io
+                    if ::IO.select [io], nil, nil, Pcap::TSHARK_READ_TIMEOUT
+                        return io.readline.chomp
+                    end
+
+                    raise Errno::ETIMEDOUT, "read timed out"
+                end
+
+                def self.next_from_io io
+                    if line = readline(io)
+                        fields = line.split SEPARATOR, FIELD_COUNT
+                        hash = {}
+                        FIELDS.each do |key|
+                            val = fields.shift
+                            hash[key] = val.empty? ? nil : val
+                        end
+                        return hash
+                    end
+                rescue Exception => e
+                    Pcap.warning e.message
+                    return nil
+                end
+
+            end
+        end
+    end
+end

data/lib/woolen_common/pcap/mu/scenario/pcap/rtp.rb

@@ -0,0 +1,72 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+    class Scenario
+        module Pcap
+            module Rtp
+
+                TRUNC_COUNT = 5
+
+                def self.preprocess packets, fields_per_packet
+                    signaled_by = []
+                    prev_signal_frame = {}
+                    packets.each_with_index do |packet, idx|
+                        fields = fields_per_packet[idx]
+                        if fields and fields[:rtp]
+                            flow_id = packet.flow_id
+                            if frame = fields[:"rtp.setup-frame"]
+                                prev_signal_frame[flow_id] = frame
+                            else
+                                if frame = prev_signal_frame[flow_id]
+                                    fields[:"rtp.setup-frame"] = frame
+                                else
+                                    packets[idx] = nil
+                                    fields_per_packet[idx] = nil
+                                    next
+                                end
+                            end
+                            sig_idx = frame.to_i
+                            signaled_by[idx] = sig_idx
+                        end
+                    end
+
+                    flow_to_count = Hash.new 0
+                    prev_setup_frame = {}
+                    keep_frames = []
+                    packets.each_with_index do |packet, idx|
+                        if setup_frame = signaled_by[idx]
+                            flow = packet.flow_id
+                            count = flow_to_count[flow]
+                            if setup_frame != prev_setup_frame[flow]
+                                prev_setup_frame[flow] = setup_frame
+                                count = 1
+                            else
+                                count += 1
+                            end
+                            if count <= TRUNC_COUNT
+                                keep_frames << idx + 1
+                            else
+                                packets[idx] = nil
+                                fields_per_packet[idx] = nil
+                            end
+                            flow_to_count[flow] = count
+                        end
+                    end
+
+                    packets.reject! { |p| not p }
+                    fields_per_packet.reject! { |p| not p }
+
+                    filter = "not rtp"
+                    keep_frames.each do |frame|
+                        filter << " or frame.number == #{frame}"
+                    end
+
+                    return filter
+                end
+            end
+        end
+    end
+end

data/lib/woolen_common/pcap/pcap.rb

@@ -0,0 +1,115 @@
+# -*- encoding : utf-8 -*-
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+require 'socket'
+require 'stringio'
+require 'mu/fixnum_ext'
+
+module Mu
+
+    class Pcap
+        class ParseError < StandardError;
+        end
+
+        LITTLE_ENDIAN = 0xd4c3b2a1
+        BIG_ENDIAN = 0xa1b2c3d4
+
+        DLT_NULL = 0
+        DLT_EN10MB = 1
+        DLT_RAW = 12 # DLT_LOOP in OpenBSD
+        DLT_LINUX_SLL = 113
+
+        attr_accessor :header, :pkthdrs
+
+        def initialize
+            @header = Header.new
+            @pkthdrs = []
+        end
+
+        # Read PCAP file from IO and return Mu::Pcap.  If decode is true, also
+        # decode the Pkthdr packet contents to Mu::Pcap objects.
+        def self.read io, decode=true
+            pcap = Pcap.new
+            pcap.header = each_pkthdr(io, decode) do |pkthdr|
+                pcap.pkthdrs << pkthdr
+            end
+            return pcap
+        end
+
+        # Create PCAP from list of packets.
+        def self.from_packets packets
+            pcap = Pcap.new
+            packets.each do |packet|
+                pkthdr = Mu::Pcap::Pkthdr.new
+                pkthdr.pkt = packet
+                pcap.pkthdrs << pkthdr
+            end
+            return pcap
+        end
+
+        # Write PCAP file to IO.  Uses big-endian and linktype EN10MB.
+        def write io
+            @header.write io
+            @pkthdrs.each do |pkthdr|
+                pkthdr.write io
+            end
+        end
+
+        # Read PCAP packet headers from IO and return Mu::Pcap::Header.  If decode
+        # is true, also decode the Pkthdr packet contents to Mu::Pcap objects.  Use
+        # this for large files when each packet header can processed independently
+        # - it will perform better.
+        def self.each_pkthdr io, decode=true
+            header = Header.read io
+            while not io.eof?
+                pkthdr = Pkthdr.read io, header.magic
+                if decode
+                    pkthdr.decode! header.magic, header.linktype
+                end
+                yield pkthdr
+            end
+            return header
+        end
+
+        # Read packets from PCAP
+        def self.read_packets io, decode=true
+            packets = []
+            each_pkthdr(io) { |pkthdr| packets << pkthdr.pkt }
+            return packets
+        end
+
+        # Assertion used during Pcap parsing
+        def self.assert cond, msg
+            if not cond
+                raise ParseError, msg
+            end
+        end
+
+        # Warnings from Pcap parsing are printed using this method.
+        def self.warning msg
+            $stderr.puts "WARNING: #{msg}"
+        end
+
+        def == other
+            return self.class == other.class &&
+                self.header == other.header &&
+                self.pkthdrs == other.pkthdrs
+        end
+    end
+
+end
+
+require 'mu/pcap/header'
+require 'mu/pcap/pkthdr'
+require 'mu/pcap/packet'
+require 'mu/pcap/ethernet'
+require 'mu/pcap/ip'
+require 'mu/pcap/ipv4'
+require 'mu/pcap/ipv6'
+require 'mu/pcap/tcp'
+require 'mu/pcap/udp'
+require 'mu/pcap/sctp'