pcapr-local 0.1.10

data/lib/pcapr_local/www/static/style/tipsy.css

@@ -0,0 +1,7 @@
+.tipsy { padding: 5px; font-size: 10px; opacity: 0.8; filter: alpha(opacity=80); background-repeat: no-repeat;  background-image: url(tipsy.gif); }
+  .tipsy-inner { padding: 5px 8px 4px 8px; background-color: black; color: white; max-width: 200px; text-align: center; }
+  .tipsy-inner { -moz-border-radius:3px; -webkit-border-radius:3px; }
+  .tipsy-north { background-position: top center; }
+  .tipsy-south { background-position: bottom center; }
+  .tipsy-east { background-position: right center; }
+  .tipsy-west { background-position: left center; }

data/lib/pcapr_local/www/templates/browse.services.template

@@ -0,0 +1,10 @@
+<div class="cloud" style="margin-top:10px;margin-bottom:10px">
+    <% closet.render.cloud(kvs, function(kv, size) { %>
+        <span style="font-size:<%= size %>px">
+            <a class="term" title="<%= kv.key %>" href="#/browse/service/<%= escape(kv.key) %>">
+                <%= closet.util.escapeHTML(kv.key) %>
+            </a>
+            <sup style="font-size:10px"><%= kv.value %></sup>
+        </span>
+    <% }); %>
+</div>

data/lib/pcapr_local/www/templates/browse.template

@@ -0,0 +1,77 @@
+<% if (pcaps.filter) {%>
+<span style="float:right">
+    <img class="icon" src="/static/image/16x16/Cancel.png"/>
+    <a href="#/browse">remove filter</a>
+</span>
+<% } %>
+
+<ul id="p-main" class="s-result pcaps" style="margin-left: -40px">
+    <% $.each(pcaps.rows.slice(0, Math.min(closet.api.PAGE_SIZE, pcaps.rows.length)), function(i, row) { %>
+    <li class="l0">
+        <div class="p-body">
+            <% if (row.doc.index) { %>
+                <% var paths = row.doc.filename.split('/'); %>
+                <a class="term" href="#/browse/dir/">/</a>
+                <% $.each(paths.slice(0, paths.length-1), function(k, path) {%>
+                    <a class="term" href="#/browse/dir/<%= escape(paths.slice(0,k+1).join('/')) %>">
+                        <%= closet.util.escapeHTML(path) %>
+                    </a> /
+                <% }); %>
+                <a href="#/browse/pcap/<%= escape(row.id) %>"><%= closet.util.escapeHTML(paths[paths.length-1]) %></a>
+            <% } else if (row.doc.status === 'indexing') { %>
+                <span class="throbber"/>
+                <span style="font-size:125%;font-weight:bold;color:#e66c25">
+                    <%= closet.util.escapeHTML(row.doc.filename) %>
+                </span>
+            <% } else if (row.doc.status === 'failed' || row.doc.status === 'aborted') { %>
+                <a id="<%= row.doc._id %>" class="remove" href="javascript:void(0)">
+                    <img class="icon" src="/static/image/16x16/Cancel.png"/>
+                </a>
+                <span style="font-size:125%;font-weight:bold;color:#dd1122">
+                    <%= closet.util.escapeHTML(row.doc.filename) %>
+                </span>
+            <% } else {%>
+                <span style="font-size:125%;font-weight:bold;color:grey">
+                    <%= closet.util.escapeHTML(row.doc.filename) %>
+                </span>
+            <% } %>
+        </div>
+        <% if (row.doc.index) { %>
+            <div class="p-protos">
+                <span>services: </span>
+                <% var services = row.doc.index.services.sort(); %>
+                <% $.each(services.slice(0,10), function(j, service) { %>
+                    <a class="meta-proto" href="#/browse/service/<%= escape(service) %>">
+                        <%= closet.util.escapeHTML(service) %>
+                    </a>
+                <% }); %>
+                <% if (services.length > 10) { %>
+                    <span>(<%= services.length-10 %> more)</span>
+                <% } %>
+            </div>
+        <% } %>            
+        <div style="font-size: smaller">
+            <a href="#/browse/status/<%= escape(row.doc.status) %>">
+                <%= closet.util.escapeHTML(row.doc.status) %>
+            </a>, 
+            <%= closet.quantity.timespan(new Date(row.doc.created_at)) %>, 
+            <%= closet.quantity.bytes(row.doc.stat.size) %>
+            <% if (row.doc.index) { %>
+                <span>, 
+                <%= closet.quantity.count(row.doc.index.about.flows,'flow') %>, 
+                <%= closet.quantity.count(row.doc.index.about.packets,'packet') %>,
+                <%= row.doc.index.about.duration %> seconds
+                </span>
+            <% } %>
+        </div>
+    </li>
+    <% }); %>
+</ul>
+
+<% if (pcaps.rows.length === closet.api.PAGE_SIZE+1) { %>
+    <div style="margin-left: 20px">
+        <span>&raquo;&nbsp;</span>
+        <% var last = pcaps.rows[pcaps.rows.length-1]; %>
+        <a href="<%= path %>?nextkey=<%= escape(JSON.stringify(last.key)) %>&nextid=<%= escape(last.id) %>">more</a>
+    </div>
+<% } %>

data/lib/pcapr_local/www/templates/flows.template

@@ -0,0 +1,38 @@
+<div>
+<% if (flows.length > 0) { %>
+<span class="info">
+    We found <%= closet.quantity.count(flows.length, 'candidate flow') %>. 
+    Select one to create a scenario!
+</span>
+<table id="flows-table">
+    <colgroup span="8"/>
+    <colgroup span="1" style="width:100%"/>
+    <thead>
+        <tr>
+            <td>id</td>
+            <td><span style="visibility:hidden">&#9660;</span><span class="time">time</span></td>
+            <td>packets</td>
+            <td><span style="visibility:hidden">&#9660;</span><span class="src">src</span></td>
+            <td>sport</td>
+            <td><span style="visibility:hidden">&#9660;</span><span class="dst">dst</span></td>
+            <td><span class="dport">dport</span></td>
+            <td><span style="visibility:hidden">&#9660;</span><span class="service">service</span></td>
+            <td>title</td>
+        </tr>
+    </thead>
+    <% $.each(flows, function(_, flow) { %>
+    <tr class="flow">
+        <td><%= flow.id %>.</td>
+        <td><span class="field" id="flow.time"><%= flow.time.toFixed(4) %></span></td>
+        <td><span class="term" href="#/packets?pkt.flow:<%= escape(flow.id) %>"><%= escape(flow.packets) %></span></td>
+        <td><span class="term field" id="flow.src"><%= closet.util.escapeHTML(flow.src) %></span></td>
+        <td><span class="term"><%= flow.sport %></span></td>
+        <td><span class="term field" id="flow.dst"><%= closet.util.escapeHTML(flow.dst) %></span></td>
+        <td><span class="term" id="flow.dport"><%= flow.dport %></span></td>
+        <td><span class="term field" id="flow.service"><%= closet.util.escapeHTML(flow.service) %></span></td>
+        <td><a href="#/browse/pcap/<%= escape(pcap._id) %>/flow/<%= escape(flow.id) %>"><%= closet.util.escapeHTML(flow.title) %></a></td>
+    </tr>
+    <% }); %>
+</table>
+<% } %>
+</div>

data/lib/pcapr_local/www/templates/pcap.template

@@ -0,0 +1,63 @@
+<div style="margin-top:20px;margin-bottom:20px">
+<div class="p-body">
+    <% if (pcap.index) { %>
+        <% var paths = pcap.filename.split('/'); %>
+        <a class="term" href="#/browse/dir/">/</a>
+        <% $.each(paths.slice(0, paths.length-1), function(k, path) {%>
+            <a class="term" href="#/browse/dir/<%= escape(paths.slice(0,k+1).join('/')) %>">
+                <%= closet.util.escapeHTML(path) %>
+            </a> /
+        <% }); %>
+        <a class="extern" href="/pcaps/1/pcap/<%= escape(pcap._id) %>">
+            <%= closet.util.escapeHTML(paths[paths.length-1]) %>
+        </a>
+    <% } else if (pcap.status === 'indexing') { %>
+        <span class="throbber"/>
+        <span style="font-size:125%;font-weight:bold;color:#e66c25">
+            <%= closet.util.escapeHTML(pcap.filename) %>
+        </span>
+    <% } else if (pcap.status === 'failed') { %>
+        <span style="font-size:125%;font-weight:bold;color:#dd1122">
+            <%= closet.util.escapeHTML(row.doc.filename) %>
+        </span>
+    <% } else {%>
+        <span style="font-size:125%;font-weight:bold;color:grey">
+            <%= closet.util.escapeHTML(row.doc.filename) %>
+        </span>
+    <% } %>
+</div>
+<div style="font-size: smaller">
+    <a href="#/browse/status/<%= escape(pcap.status) %>"><%= pcap.status %></a>, 
+    <%= closet.quantity.timespan(new Date(pcap.created_at)) %>, 
+    <%= closet.quantity.bytes(pcap.stat.size) %>
+    <% if (pcap.index) { %>
+        <span>, 
+        <%= closet.quantity.count(pcap.index.about.flows,'flow') %>, 
+        <%= closet.quantity.count(pcap.index.about.packets,'packet') %>,
+        <%= pcap.index.about.duration %> seconds
+        </span>
+    <% } %>
+</div>
+<p/>
+<div class="reports">
+    <span class="showonload" style="display:none">
+        Reports:&nbsp;<select class="reports" style="padding-right:20px"></select>
+    </span>
+    <h3 class="title"></h3>
+    <div class="settings" style="display:none; margin-top:-10px">
+        <span style="color:#105892">&#9658;</span>
+        <a class="toggler" href="javascript:void(0)">Settings</a>
+        <div class="options" style="display:none;margin-left:20px;border-left:2px dotted #d0d0d0;padding-left:10px">
+        </div>
+    </div>
+    <div class="report" style="margin-top:20px">
+        <p/><span class="throbber">analyzing</span>
+    </div>
+</div>
+
+<div style="font-size: smaller">
+        Download a
+        <a target="_none" href="/pcaps/1/export_to_par/<%= escape(pcap._id) %>">PAR file </a>
+        .
+</div>
+</div>

data/lib/pcapr_local/www/templates/sip.calls.template

@@ -0,0 +1,35 @@
+<div>
+    <% $.each(calls, function(_, call) { %>
+        <div>
+            Call <span style="font-weight:bold"><%= closet.util.escapeHTML(call.src) %></span>
+            &nbsp;&raquo;&nbsp;
+            <span style="font-weight:bold"><%= closet.util.escapeHTML(call.dst) %></span>
+            &nbsp;
+            [<span style="font-size:smaller"><%= closet.util.escapeHTML(call.id) %></span>]
+        </div>
+        <div style="margin-left: 20px">
+            This call was made from <span style="font-weight:bold"><%= closet.util.escapeHTML(call.from) %></span> to 
+            <span style="font-weight:bold"><%= closet.util.escapeHTML(call.to) %></span> and is made up 
+            of <%= closet.quantity.count(call.flows.length,'flow') %> and
+            <%= closet.quantity.count(call.packets,'packet') %>.
+            
+            <% if (call.rtp.length > 0) { %>
+                We also found <%= closet.quantity.count(call.rtp.length,'RTP stream') %> in this call 
+                with the following media types:
+                <ul>
+                <% $.each(call.media, function(_, media) {%>
+                    <li style="list-style: disc outside !important">
+                        <%= closet.util.escapeHTML(media) %>
+                    </li>
+                <% }); %>
+                </ul>
+            <% } %>
+
+            <div>
+            <img class="icon" src="/static/image/16x16/Download.png"/>
+            Download <a href="/pcaps/1/pcap/<%= escape(pcap._id) %>/api/packets/slice?q=<%=escape(call.q)%>">pcap</a> for this call.
+            </div>
+            <p/>
+        </div>
+    <% }); %>
+</div>

data/lib/pcapr_local/www/templates/statistics.template

@@ -0,0 +1,6 @@
+<div>
+    <%= stats.pcaps || 0 %> pcaps, 
+    <%= stats.flows || 0 %> flows,
+    <%= stats.packets || 0 %> packets,
+    <%= closet.quantity.bytes(stats.bytes || 0) %>
+</div>

data/lib/pcapr_local/xtractr.rb

@@ -0,0 +1,179 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'thread'
+require 'net/http'
+require 'uri'
+
+module PcaprLocal
+class Xtractr
+    class XtractrError < StandardError; end
+
+    EXE_PATH = File.join(ROOT, 'lib/exe/xtractr')
+    
+    def initialize config
+        @xtractr_path = EXE_PATH
+        @idle_timeout = config.fetch("idle_timeout")
+        @index_dir    = config.fetch("index_dir")
+        @reaper_interval = config.fetch("reaper_interval", REAPER_INTERVAL)
+        # Hash of index dir to xtractr instance.
+        @xcache = {}
+        # Lock to synchronize creation/destruction of xtractr instances.
+        @xcache_lock = Mutex.new
+
+        start_reaper
+        at_exit do
+            shutdown
+        end 
+    end
+
+    # Idle xtractr process reaper runs every REAPER_INTERVAL seconds.
+    REAPER_INTERVAL = 10 
+
+    # Start reaper thread.
+    def start_reaper
+        Thread.new do
+            loop do
+                begin
+                    reap
+                rescue Exception
+                    Logger.error "Exception while cleaning up idle processes: #{e.message}\n" + e.backtrace.join("\n")
+                end
+                sleep @reaper_interval
+            end
+        end
+    end
+
+    # Kill xtractr instances at exit (idle or not).
+    def shutdown
+        @xcache.each_value do |xtractr|
+            $stderr.puts "stopping xtractr process"
+            xtractr.stop rescue nil
+        end
+    end
+
+    # Kills idle xtractr processes
+    def reap
+        @xcache_lock.synchronize do 
+            @xcache.to_a.each do |dir, xtractr|
+                if xtractr.lock.try_lock
+                    # xtractr is not in use right now
+                    begin
+                        if xtractr.last_use + @idle_timeout < Time.new.to_f
+                            xtractr.stop
+                            @xcache.delete dir
+                        end
+                    ensure
+                        xtractr.lock.unlock
+                    end
+                end
+            end
+        end
+    end
+
+    # Does path look like an xtractr index directory?
+    def self.index_dir?(path)
+        File.exist? File.join(path, 'packets.db')
+    end
+
+    # Returns timestamp (float) for xtractr index.
+    def self.index_time(path)
+        db_file = File.join(path, 'packets.db')
+        if File.exists? db_file
+            File.mtime(db_file).to_f
+        elsif File.exists? path
+            File.mtime(path).to_f
+        else
+            0.0
+        end
+    end
+
+    class XtractrIndexingException < XtractrError; end
+
+    # Last line of normal xtractr indexing output.
+    RE_INDEXING_DONE = /optimizing terms\.db\.\.\.done/
+
+    # Indexes pcap it index_dir and returns hash containing xtractr summary data.
+    # Raises exception if indexing fails.
+    def index pcap_path, index_dir
+        FileUtils.mkdir_p index_dir
+
+        command = [@xtractr_path, 'index', index_dir, '--mode', 'forensics', pcap_path]
+        Logger.debug "running: #{command.inspect}"
+        xtractr_out = Tempfile.new "xtractr_out"
+        pid = fork do
+            # Xtractr forks a child process. Set process group so we
+            # can send signal to all processes in a group.
+            STDOUT.reopen xtractr_out
+            STDERR.reopen xtractr_out
+            exec *command 
+        end
+        #XXX enforce timeout.
+        Process.wait pid 
+
+        xtractr_out.rewind
+        output = xtractr_out.read
+
+        unless $?.exitstatus == 0 and Xtractr.index_dir?(index_dir) and output =~ RE_INDEXING_DONE
+            Logger.error "Indexing failed with output:\n" + output
+            raise XtractrIndexingException, "Indexing failed"
+        end
+
+        return get_summary(index_dir)
+    ensure
+        xtractr_out.close! if xtractr_out
+    end
+
+    def get_summary index_dir
+        # Start xtractr in browse mode and get summary data
+        browser = Instance.new index_dir, @xtractr_path
+        about = JSON.parse(browser.get('api/about')[2])
+        services = JSON.parse(browser.get('api/services')[2])
+        service_names = []
+        services["rows"].each do |row|
+            service_names << row['name'].downcase
+        end
+        return { :about => about, :services => service_names }
+    ensure
+        browser.stop if browser
+    end
+    
+    # Forwards GET request to xtractr instance created for index_dir.
+    # A relative path will be expanded relative to the configured index_dir.
+    def get index_dir, url
+        xtractr = nil
+        xtractr = xtractr_for index_dir
+        xtractr.get url
+    end
+
+    # Forwards POST request to xtractr instance created for index_dir.
+    # A relative path will be expanded relative to the configured index_dir.
+    def post index_dir, url, body
+        xtractr = nil
+        xtractr = xtractr_for index_dir
+        xtractr.post url, body
+    end
+
+    def xtractr_for index_dir
+        # Ensure index_dir path is absolute.
+        if index_dir.slice(0,1) != '/'
+            index_dir = File.expand_path(File.join(@index_dir, index_dir))
+        end
+
+        # Get or create xtractr instance.
+        @xcache_lock.synchronize do
+            xtractr = @xcache[index_dir]
+            if not xtractr
+                xtractr = Instance.new(index_dir, @xtractr_path)
+                @xcache[index_dir] = xtractr
+            end
+            return xtractr
+        end
+    end
+
+end
+end
+
+
+require 'pcapr_local/xtractr/instance'

data/lib/pcapr_local/xtractr/instance.rb

@@ -0,0 +1,172 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'thread'
+require 'net/http'
+require 'uri'
+
+module PcaprLocal
+class Xtractr
+class Instance
+    attr_reader :last_use, :lock, :pid, :port
+
+    def initialize index_dir, xtractr_path
+        @index_dir = index_dir
+        @xtractr_path = xtractr_path
+        @lock = Mutex.new
+        @last_use = Time.new.to_f # for idle timeouts
+        @pid = nil
+        @port = nil
+    end
+
+    # Does GET request and returns response headers and body.
+    def get path_and_params
+        start if not @pid
+        @lock.synchronize do
+            @last_use = Time.new.to_f
+
+            # Make request to xtractr
+            uri = URI.parse("http://127.0.0.1:#{@port}/#{path_and_params}")
+            response = Net::HTTP.get_response(uri)
+
+            # Copy headers from response
+            headers = {}
+            response.each_header {|name,val| headers[name] = val}
+
+            return response.code.to_i, headers, response.body
+        end
+    end
+
+    # Does POST request and returns response headers and body.
+    def post path_and_params, post_body
+        start if not @pid
+        @lock.synchronize do
+            @last_use = Time.new.to_f
+
+            # Make request to xtractr
+            Net::HTTP.start('localhost', @port) do |http|
+                http.request_post "/#{path_and_params}", post_body do |response|
+                    headers = {}
+                    response.each_header {|name,val| headers[name] = val}
+                    return response.code.to_i, headers, response.body
+                end
+            end
+        end
+    end
+
+    # Starts underlying process.
+    def start
+        @lock.synchronize do 
+            return if @pid
+            err = nil
+            # There is a remote possibility that the random port we pick will be
+            # in use at the moment we try to bind to it. Thus the retries.
+            3.times do  |n|
+                begin
+                    do_start
+                    return
+                rescue => err
+                end
+            end
+            raise err
+        end
+    end
+
+    class XtractrStartupException < XtractrError; end
+
+    # Start xtractr
+    MAX_START_TIME = 30 
+    RE_STARTED = /starting on http:/i
+    def do_start 
+        port = Instance.free_local_port
+        command = [@xtractr_path, 'browse', @index_dir, '--port', port.to_s]
+        Logger.debug "running: #{command.inspect}"
+        xtractr_out = Tempfile.new "xtractr_out"
+        pid = fork do
+            # Xtractr forks a child process. Set process group so we
+            # can send signal to all processes in a group.
+            Process.setpgid $$, $$
+            STDOUT.reopen xtractr_out
+            STDERR.reopen xtractr_out
+            Dir.chdir @index_dir
+            exec *command 
+        end
+
+        begin
+            Timeout.timeout MAX_START_TIME do 
+                # Wait for "starting" line.
+                while xtractr_out.grep(/starting on http:\/\/127\.0\.0\.1:#{port}/i).empty?
+                    xtractr_out.rewind
+                    sleep 0.01
+                end
+                # Sanity check that server is up.
+                Net::HTTP.start('127.0.0.1', port) do |http|
+                    http.options("/")
+                end
+            end
+        rescue Timeout::Error, SystemCallError
+            Logger.error "Xtractr failed to start on port #{port}"
+            xtractr_out.rewind
+            Logger.error "Xtractr output: #{xtractr_out.read.inspect}"
+            kind_kill -pid
+            raise XtractrStartupException, "Timeout waiting for xtractr to startup"
+        end
+        
+        @last_use = Time.new.to_f
+        @pid = pid
+        @port = port
+    end
+
+    # Kills underlying xtractr process.
+    def stop
+        if @pid
+            kind_kill -@pid
+            @pid = nil
+        end
+    end
+
+    # Sends SIGTERM and waits for process to exit. If process does
+    # not exit within wait period, sends SIGKILL. Use a negative
+    # pid to kill all members of a process group.
+    def kind_kill pid, wait=1
+        begin
+            Process.kill 15, pid
+            # Wait for process (or all group members) to
+            # exit.
+            Timeout.timeout wait do
+                loop { Process.wait(pid) }
+            end
+        rescue Errno::ECHILD, Errno::ESRCH
+            # Processes is dead or group has no members.
+            return 
+        rescue Timeout::Error, StandardError
+            # Process did not shutdown in time (or there
+            # was an unexpected error).
+            Process.kill(9, pid) rescue nil
+        end
+    end
+    
+    # Pick a random port over 10000 and verify that it can be bound to (on localhost)
+    def self.free_local_port
+        while true
+            port = rand(0xffff-10000) + 10000
+            socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+            socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
+            addr = Socket.pack_sockaddr_in(port, '127.0.0.1')
+            begin
+                socket.bind addr
+                return port
+            rescue Errno::EADDRINUSE
+                next # port in use
+            ensure
+                socket.close
+            end
+        end
+    end
+
+end
+end
+end
+
+