pcapr-local 0.1.10

data/lib/mu/pcap/udp.rb

@@ -0,0 +1,69 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Pcap
+
+class UDP < Packet
+    attr_accessor :src_port, :dst_port
+
+    def initialize src_port=0, dst_port=0
+        super()
+        @src_port = src_port
+        @dst_port = dst_port
+    end
+
+    def flow_id
+        return [:udp, @src_port, @dst_port]
+    end
+
+    FMT_nnnn = 'nnnn'
+    def self.from_bytes bytes
+        bytes_length = bytes.length
+        bytes_length >= 8 or
+            raise ParseError, "Truncated UDP header: expected 8 bytes, got #{bytes_length} bytes"
+        sport, dport, length, checksum = bytes.unpack(FMT_nnnn)
+        bytes_length >= length or 
+            raise ParseError, "Truncated UDP packet: expected #{length} bytes, got #{bytes_length} bytes"
+        udp = UDP.new sport, dport
+        udp.payload_raw = bytes[8..-1]
+        udp.payload = bytes[8..length]
+        return udp
+    end
+
+    def write io, ip
+        length = @payload.length
+        length_8 = length + 8
+        if length_8 > 65535
+            Pcap.warning "UDP payload is too large"
+        end
+        pseudo_header = ip.pseudo_header length_8
+        header = [@src_port, @dst_port, length_8, 0] \
+            .pack FMT_nnnn
+        checksum = IP.checksum(pseudo_header + header + @payload)
+        header = [@src_port, @dst_port, length_8, checksum] \
+            .pack FMT_nnnn
+        io.write header
+        io.write @payload
+    end
+
+    def self.udp? packet
+        return packet.is_a?(Ethernet) &&
+            packet.payload.is_a?(IP) &&
+            packet.payload.payload.is_a?(UDP)
+    end
+
+    def to_s
+        return "udp(%d, %d, %s)" % [@src_port, @dst_port, @payload.inspect]
+    end
+
+    def == other
+        return super &&
+            self.src_port == other.src_port &&
+            self.dst_port == other.dst_port
+    end
+end
+
+end
+end

data/lib/mu/scenario/pcap.rb

@@ -0,0 +1,164 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'tempfile'
+require 'fileutils'
+require 'mu/scenario/pcap/fields'
+require 'mu/pcap'
+require 'json'
+
+module Mu
+class Scenario
+
+module Pcap
+    TSHARK_READ_TIMEOUT     = 10.0 # seconds
+    TSHARK_LINES_PER_PACKET = 16384
+    TSHARK_OPTS = "-n -o tcp.desegment_tcp_streams:false"
+    TSHARK_SIZE_OPTS = "-n -o 'column.format: cum_size, \"%B\"'"
+    TSHARK_PSML_OPTS = %Q{#{TSHARK_OPTS} -o 'column.format: "Protocol", "%p", "Info", "%i"'}
+
+    MAX_PCAP_SIZE = 102400 # 100KB
+    MAX_RAW_PCAP_SIZE_MB = 25
+    MAX_RAW_PCAP_SIZE = MAX_RAW_PCAP_SIZE_MB * 1024 * 1000
+    EXCLUDE_FROM_SIZE_CHECK = ['rtp'].freeze
+
+    class PcapTooLarge < StandardError; end
+
+    def self.validate_pcap_size(path)
+        tshark_filter = EXCLUDE_FROM_SIZE_CHECK.map{ |proto| "not #{proto}" }.join " and "
+        io = ::IO.popen "tshark #{TSHARK_SIZE_OPTS} -r #{path} -R '#{tshark_filter}' | tail -1"
+        if ::IO.select [ io ], nil, nil, TSHARK_READ_TIMEOUT
+            if io.eof?
+                size = 0
+            else
+                last_line = io.readline
+                size = last_line.to_i
+            end
+        end
+
+        if size.nil? or size == 0
+            size = File.size(path)
+        end
+
+        if size > MAX_PCAP_SIZE
+            raise PcapTooLarge, "Selected packets have a size of #{size} bytes which " +
+                "exceeds the #{MAX_PCAP_SIZE} byte maximum."
+        end
+
+        if size > MAX_RAW_PCAP_SIZE
+            raise PcapTooLarge, "Selected packets have a raw size of #{size} bytes which " +
+                "exceeds the #{MAX_RAW_PCAP_SIZE_MB}MB maximum."
+        end
+
+        return size
+    end
+
+    PAR_VERSION = 1
+    def self.export_to_par pcap_path, opts=nil
+        opts ||= {}
+
+        # Open pcap file
+        File.exist?(pcap_path) or raise "Cannot open file '#{pcap_path}'."
+        validate_pcap_size pcap_path
+        pcap = open pcap_path, 'rb'
+
+        # Get Mu::Pcap::Packets
+        packets = to_pcap_packets pcap, opts[:isolate_l7]
+
+        # Write normalized packets to tempfile
+        tmpdir = Dir.mktmpdir
+        norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+        pcap = Mu::Pcap.from_packets packets
+        pcap.write norm_pcap
+        norm_pcap.close
+
+        # Get wireshark dissected field values for all packets.
+        `tshark -T fields #{TSHARK_OPTS} #{Fields::TSHARK_OPTS} -Eseparator='\xff' -r #{norm_pcap.path} > #{tmpdir}/fields`
+        fields = open "#{tmpdir}/fields", 'rb'
+
+        # Get wireshark dissected field values for all packets.
+        fields_array = []
+        if fields
+            packets.each do |packet|
+                fields_array <<  Fields.next_from_io(fields)
+            end
+        end
+
+        # Protocol specific preprocessing, packets may be deleted.
+        Rtp.preprocess packets, fields_array
+
+        File.open "#{tmpdir}/packets.dump", 'wb' do |f|
+            Marshal.dump packets, f
+        end
+
+        # Create a second pcap with packets removed.
+        norm_pcap = File.open "#{tmpdir}/normalized.pcap", 'wb'
+        pcap = Mu::Pcap.from_packets packets
+        pcap.write norm_pcap
+        norm_pcap.close
+
+        # Dump PSML to file.
+        # (The no-op filter sometimes produces slighty more verbose descriptions.)
+        `tshark -T psml #{TSHARK_PSML_OPTS} -r #{norm_pcap.path} -R 'rtp or not rtp' > #{tmpdir}/psml`
+
+        # Dump PDML io file.
+        `tshark -T pdml #{TSHARK_OPTS} -r #{norm_pcap.path} > #{tmpdir}/pdml`
+        pdml = open "#{tmpdir}/pdml", 'rb'
+
+        # Create about
+        open "#{tmpdir}/about", 'w' do |about|
+            about.puts({:par_version => PAR_VERSION}.to_json)
+        end
+
+        # Create zip
+        Dir.chdir tmpdir do
+            system "zip -q dissected.zip about pdml psml fields packets.dump normalized.pcap"
+            return open("dissected.zip")
+        end
+    ensure
+        if tmpdir
+            FileUtils.rm_rf tmpdir
+        end
+    end
+
+    def self.to_pcap_packets io, isolate_l7=true
+        packets = []
+
+        # Read Pcap packets from Pcap
+        Mu::Pcap.each_pkthdr io do |pkthdr|
+            if pkthdr.len != pkthdr.caplen
+                raise Mu::Pcap::ParseError, "Error: Capture contains truncated packets. " +
+                    "Try recapturing with an increased snapshot length."
+            end
+            if not pkthdr.pkt.is_a? Mu::Pcap::Ethernet
+                warning 'Unable to parse packet, skipping.'
+            end
+            packets << pkthdr.pkt
+        end
+
+        if (packets.length == 0)
+            raise Mu::Pcap::ParseError, "No valid packets found!"
+        end
+
+        packets = Mu::Pcap::IPv4.reassemble packets
+
+        if isolate_l7
+            packets = Mu::Pcap::Packet.isolate_l7 packets
+        end
+
+        packets = Mu::Pcap::Packet.normalize packets
+        packets = Mu::Pcap::TCP.split packets
+
+        packets
+    end
+
+    def self.warning msg
+        $stderr.puts "WARNING: #{msg}"#, caller, $!
+    end
+end
+
+end
+end
+
+require 'mu/scenario/pcap/rtp'

data/lib/mu/scenario/pcap/fields.rb

@@ -0,0 +1,50 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'mu/scenario/pcap'
+
+module Mu
+class Scenario
+module Pcap
+
+class Fields
+    FIELDS = [
+        :rtp,
+        :"rtp.setup-frame"
+    ].freeze
+    FIELD_COUNT = FIELDS.length
+    SEPARATOR   = "\xff".freeze
+    TSHARK_OPTS = "-Eseparator='#{SEPARATOR}'" 
+    FIELDS.each do |field|
+        TSHARK_OPTS << " -e #{field}"
+    end
+    TSHARK_OPTS.freeze
+
+    def self.readline io
+        if ::IO.select [ io ], nil, nil, Pcap::TSHARK_READ_TIMEOUT
+            return io.readline.chomp
+        end 
+        
+        raise Errno::ETIMEDOUT, "read timed out"
+    end
+
+    def self.next_from_io io
+        if line = readline(io)
+            fields = line.split SEPARATOR, FIELD_COUNT
+            hash = {}
+            FIELDS.each do |key|
+                val = fields.shift
+                hash[key] = val.empty? ? nil : val
+            end
+            return hash
+        end
+    rescue Exception => e
+        Pcap.warning e.message
+        return nil
+    end
+
+end
+end
+end
+end

data/lib/mu/scenario/pcap/rtp.rb

@@ -0,0 +1,71 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+module Mu
+class Scenario
+module Pcap
+module Rtp
+
+    TRUNC_COUNT = 5
+
+    def self.preprocess packets, fields_per_packet
+        signaled_by = []
+        prev_signal_frame = {}
+        packets.each_with_index do |packet, idx|
+            fields = fields_per_packet[idx]
+            if fields and fields[:rtp]
+                flow_id = packet.flow_id
+                if frame = fields[:"rtp.setup-frame"]
+                    prev_signal_frame[flow_id] = frame
+                else
+                    if frame = prev_signal_frame[flow_id]
+                        fields[:"rtp.setup-frame"] = frame
+                    else
+                        packets[idx] = nil
+                        fields_per_packet[idx] = nil
+                        next
+                   end
+                end
+                sig_idx = frame.to_i 
+                signaled_by[idx] = sig_idx
+            end
+        end
+
+        flow_to_count = Hash.new 0
+        prev_setup_frame = {} 
+        keep_frames = []
+        packets.each_with_index do |packet, idx|
+            if setup_frame = signaled_by[idx]
+                flow = packet.flow_id
+                count = flow_to_count[flow]
+                if setup_frame != prev_setup_frame[flow]
+                    prev_setup_frame[flow] = setup_frame
+                    count = 1
+                else
+                    count += 1 
+                end
+                if count <= TRUNC_COUNT
+                    keep_frames << idx + 1
+                else
+                    packets[idx] = nil
+                    fields_per_packet[idx] = nil
+                end
+                flow_to_count[flow] = count
+            end
+        end
+
+        packets.reject! {|p| not p }
+        fields_per_packet.reject! {|p| not p }
+
+        filter = "not rtp"
+        keep_frames.each do |frame|
+            filter << " or frame.number == #{frame}"
+        end
+
+        return filter
+    end
+end
+end
+end
+end

data/lib/pcapr_local.rb

@@ -0,0 +1,159 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'rubygems'
+require 'environment'
+require 'pcapr_local/config'
+require 'pcapr_local/scanner'
+require 'pcapr_local/server'
+require 'pcapr_local/xtractr'
+require 'logger'
+
+module PcaprLocal
+    START_SCRIPT = File.join(ROOT, 'bin/pcapr_start.rb')
+
+    # We share a single Logger across all of pcapr.Local.
+    Logger = Logger.new(STDOUT)
+
+    # Recreate logger using configured log location.
+    LOGFILE = "server.log"
+    def self.start_logging log_dir
+        if log_dir and not log_dir.empty?
+            if const_defined? :Logger
+                remove_const :Logger
+            end
+
+            logfile = File.join(log_dir, LOGFILE)
+            FileUtils.mkdir_p log_dir
+            const_set :Logger, Logger.new(logfile, 5)
+        end
+    end
+
+    # Start xtractr instance manager.
+    def self.start_xtractr config
+        xtractr_config = config['xtractr'].merge(
+            "index_dir" => config.fetch("index_dir"),
+            "pcap_dir"  => config.fetch("pcap_dir")
+        )
+        Xtractr.new xtractr_config
+    end
+
+    # Start file system scanner.
+    def self.start_scanner config, db, xtractr
+        scanner_config = config.fetch('scanner').merge( 
+            "index_dir" => config.fetch("index_dir"),
+            "pcap_dir" => config.fetch("pcap_dir"),
+            "db" => db,
+            "xtractr" => xtractr
+        )
+        PcaprLocal::Scanner.start scanner_config
+    end
+
+    # Start webserver UI/API
+    def self.start_app config, db, scanner, xtractr
+        app_config = config.fetch "app"
+        root = File.expand_path(File.dirname(__FILE__))
+        app_file = File.join(root, "pcapr_local/server.rb")
+        PcaprLocal::Server.run! \
+            :app_file => app_file,
+            :dump_errors => true,
+            :logging => true,
+            :port    => app_config.fetch("port"), 
+            :bind    => app_config.fetch("host"),
+            :db      => db,
+            :scanner => scanner,
+            :xtractr => xtractr
+    end
+
+    def self.get_db config
+        PcaprLocal::DB.get_db config.fetch("couch")
+    end
+
+    def self.start config=nil
+        config ||= PcaprLocal::Config.config
+        
+        # Check that server is not already running.
+        check_pid_file config['pidfile']
+
+        # Start logging.
+        if config["log_dir"]
+            start_logging config['log_dir']
+        end
+        start_msg = "Starting server at #{config['app']['host']}:#{config['app']['port']}"
+        Logger.info start_msg
+        puts start_msg
+        puts "Log is at #{config['log_dir']}/#{LOGFILE}"
+
+        # Deamonize
+        unless config['debug_mode']
+            puts "Moving server process to the background. Run 'stoppcapr' to stop the server."
+            Process.daemon
+        end
+
+        # Create pid file that will be deleted when we shutdown.
+        create_pid_file config['pidfile']
+
+        # Get database instance
+        db = get_db config
+
+        # Xtractr manager
+        xtractr = start_xtractr config
+
+        # Start scanner thread
+        scanner = start_scanner config, db, xtractr
+
+        # Start application server
+        start_app config, db, scanner, xtractr
+    end
+
+    def self.check_pid_file file
+        if File.exist? file
+            # If we get Errno::ESRCH then process does not exist and
+            # we can safely cleanup the pid file.
+            pid = File.read(file).to_i
+            begin 
+                Process.kill(0, pid)
+            rescue Errno::ESRCH
+                stale_pid = true
+            rescue 
+            end
+
+            unless stale_pid 
+                puts "Server is already running (pid=#{pid})"
+                exit
+            end
+        end
+    end
+
+    def self.create_pid_file file
+        File.open(file, "w") { |f| f.puts Process.pid }
+
+        # Remove pid file during shutdown
+        at_exit do 
+            Logger.info "Shutting down." rescue nil
+            if File.exist? file
+                File.unlink file
+            end
+        end
+    end
+
+    # Sends SIGTERM to process in pidfile. Server should trap this
+    # and shutdown cleanly.
+    def self.stop config=nil
+        user_config = Config.user_config_path
+        if File.exist?(user_config)
+            config = PcaprLocal::Config.config user_config
+            pid_file = config["pidfile"]
+            if pid_file and File.exist? pid_file
+                pid = Integer(File.read(pid_file))
+                Process.kill -15, -pid
+            end
+        end
+    end
+end
+
+if __FILE__ == $0
+    PcaprLocal.start
+end
+