pcapr-local 0.1.10

data/lib/pcapr_local/scanner.rb

@@ -0,0 +1,250 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'find'
+require 'set'
+require 'fileutils'
+require 'tempfile'
+require 'digest/md5'
+
+module PcaprLocal
+class Scanner
+
+    # Creates scanner instance and starts it.
+    def self.start config
+        scanner = Scanner.new config
+        scanner.start
+        scanner
+    end
+
+    # Runs scanner loop in separate thread.
+    def start
+        Logger.info "Starting scanner thread"
+        Thread.new do
+            loop do 
+                begin
+                    scan
+                    @db.compact!
+                rescue Exception => e
+                    Logger.error "Exception during scanning: #{e.message}\n" + e.backtrace.join("\n")
+                end
+                sleep @scan_interval
+            end
+        end
+    end
+
+    def initialize config
+        @db = config.fetch('db')
+        @xtractr = config.fetch('xtractr')
+        @pcap_dir  = File.expand_path(config.fetch('pcap_dir'))
+        @index_dir = File.expand_path(config.fetch('index_dir'))
+        @queue_delay = config.fetch('queue_delay')
+        @scan_interval = config.fetch('interval')
+    end
+
+    # Removes doc from database and corresponding index file.
+    # Does _not_ remove original pcap.
+    def remove_doc doc
+        @db.delete_doc(doc)
+        if filename = doc['filename']
+            FileUtils.rm_f pcap_path(filename)
+            remove_index_for(filename)
+        end
+    end
+
+    def scan
+        # Get list of all pcaps
+        pcaps = self.find_pcaps
+        # Cleanup db and queue new pcaps
+        reconcile_with_db pcaps
+        # Index queued pcaps.
+        self.index
+    end
+
+    RE_PCAP = /\.p?cap\Z/
+
+    # Returns a set of pcap files (relative paths)
+    def find_pcaps
+        if not File.directory?(@pcap_dir) or not File.readable?(@pcap_dir)
+            return Set.new
+        end
+
+        pcaps = Set.new
+        pcap_prefix_size = @pcap_dir.size + 1 # /
+        Find.find @pcap_dir do |path|
+            # Don't recurse into ".pcapr_local" or other "." dirs
+            if File.basename(path) =~ /^\./
+                Find.prune
+            end
+
+            # Should be a file ending in .cap or .pcap
+            next unless path =~ RE_PCAP and File.file?(path)
+
+            rel_path = path[pcap_prefix_size..-1]
+            pcaps << rel_path
+        end
+        pcaps
+    end
+
+    def requeue_pcap rel_path
+        res = @db.view("pcaps/by_filename", :key => rel_path)
+        return nil if res['rows'].empty?
+
+        id = res['rows'][0]["id"]
+        @db.update_doc id do |doc|
+            doc['status'] = 'queued'
+            doc.delete 'index'
+            doc
+        end
+    end
+ 
+    # Adds pcap to db with status set to "queued". Returns nil w/out 
+    # updating db if the pcap was modified within the last queue_delay
+    # seconds (because pcap may not be completely copied to pcap_dir).
+    def add_pcap relative_path
+        now = Time.new
+        stat = File.stat(File.join(@pcap_dir, relative_path))
+        if now - stat.mtime < @queue_delay
+            return
+        end
+
+        # Pick determistic doc id based on path and pcap size.
+        # (for testing convenience).
+        id = Digest::MD5.new
+        id << "#{relative_path}:#{stat.size}"
+
+        doc = CouchRest::Document.new({
+            :_id => id.to_s,
+            :type => 'pcap',
+            :filename => relative_path,
+            :status => 'queued',
+            :stat => {
+                :inode => stat.ino,
+                :size => stat.size,
+                :ctime => stat.ctime,
+            },
+            :created_at => now,
+            :updated_at => now,
+        })
+        @db.save_doc doc
+
+        doc
+    end
+
+    # Indexes all documents in queue. Returns count of documents indexed.
+    def index
+        count = 0
+        @db.each_in_view("pcaps/queued", :include_docs => true) do |row|
+            index_pcap row['doc']
+            count += 1
+        end
+        count
+    end
+
+    # Creates xtractr index for pcap. Updates status from "queued" to "indexing" to "indexed".
+    # Any exception will result in a status of "failed" with the exception's message copied
+    # to the document's message attribute.
+    def index_pcap pcap
+        relative_path = pcap["filename"]
+        pcap_path = File.join(File.expand_path(@pcap_dir), relative_path)
+        index_dir = File.join(File.expand_path(@index_dir), relative_path)
+
+        # Index
+        Logger.info "Indexing #{relative_path}"
+        begin
+            @db.update_doc pcap["_id"] do |doc|
+                doc["status"] = "indexing"
+                doc
+            end
+
+            index_data = @xtractr.index pcap_path, index_dir
+
+            @db.update_doc pcap["_id"] do |doc|
+                doc['index'] = index_data
+                doc['status'] = 'indexed'
+                doc
+            end
+        rescue 
+            Logger.warn "Indexing failure: #{$!.message}"
+            @db.update_doc pcap["_id"] do |doc|
+                doc['status']  = "failed"
+                doc['message'] = $!.message
+                doc
+            end
+        end
+
+        return
+    end
+
+    def pcap_path rel_path
+        if rel_path.is_a? Hash
+            rel_path = rel_path[:filename] or raise "path not found in #{rel_path.inspect}"
+        end
+        File.expand_path File.join(@pcap_dir, rel_path)
+    end
+
+    def index_path rel_path
+        if rel_path.is_a? Hash
+            rel_path = rel_path.fetch :filename
+        end
+        File.expand_path File.join(@index_dir, rel_path)
+    end
+
+    # Because FileUtils.rm_rf is too dangerous.
+    def remove_index_for rel_path
+        target = index_path rel_path
+        if File.directory? target
+            FileUtils.rm_rf Dir.glob("#{target}/*.db")
+            FileUtils.rmdir target rescue nil
+        end
+    end
+
+    # Checks each pcap in the database, purging or requeueing documents as necessary. 
+    # Any pcaps in fs_pcaps that are not in the database are added.
+    def reconcile_with_db fs_pcaps
+        fs_pcaps = fs_pcaps.dup 
+
+        indexed = Set.new
+        @db.each_in_view("pcaps/indexed") do |row|
+            indexed << row['key']
+        end
+
+        @db.each_in_view("pcaps/by_filename") do |row|
+            path = row['key']
+
+            # Delete record if from database if pcap is not present on the 
+            # file system.
+            if not fs_pcaps.include? path
+                Logger.warn "Indexer: removing database entry for missing pcap #{path}"
+                @db.delete_doc @db.get(row['id'])
+                remove_index_for(path)
+
+                next
+            end
+
+            # Requeue pcap if xtractr index is missing or is older than the pcap.
+            if indexed.include? path
+                pcap_index_dir = File.join(@index_dir, path)
+                if not Xtractr.index_dir?(pcap_index_dir)
+                    Logger.warn "Index is missing, requeueing #{path}"
+                    requeue_pcap path
+                elsif Xtractr.index_time(pcap_index_dir) < File.mtime(pcap_path(path)).to_f
+                    Logger.info "Pcap is newer than index, requeueing #{path}"
+                    requeue_pcap path
+                end
+            end
+
+            fs_pcaps.delete path
+        end
+
+        # Remaining pcaps are unknown, add them to database
+        fs_pcaps.each do |path|
+            Logger.debug "New pcap: #{path}"
+            add_pcap path
+        end
+    end
+end
+end
+
+

data/lib/pcapr_local/server.rb

@@ -0,0 +1,178 @@
+# http://www.mudynamics.com
+# http://labs.mudynamics.com
+# http://www.pcapr.net
+
+require 'rack'
+require 'rack/contrib/jsonp'
+require 'sinatra'
+require 'pcapr_local/db'
+require 'pcapr_local/scanner'
+require 'pcapr_local/xtractr'
+require 'mu/scenario/pcap'
+
+module PcaprLocal
+class Server < Sinatra::Base
+    set :app_file, __FILE__
+    root = File.expand_path(File.dirname(__FILE__))
+    set :public, File.join(root, 'www')
+
+    use Rack::JSONP
+    mime_type :template, 'application/octet-stream'
+    mime_type :par, 'application/octet-stream'
+
+    helpers do
+        # View as object by joining keys and values.
+        # e.g. {"key" => "foo", "value" => "bar"}
+        # becomes 
+        # {"foo" => "bar"}
+        def as_object result
+            obj = {}
+            result['rows'].each do |row|
+                key = row['key']
+                if key.is_a? Array
+                    # behave like javascript Array.toString
+                    key = key.join ","
+                end
+
+                val = row['value']
+                if val.is_a? Array
+                    # behave like javascript Array.toString
+                    key = key.join ","
+                end
+
+                obj[key] = val
+            end
+            obj
+        end
+    end
+
+    # Main page.
+    get '/' do
+        redirect '/home/index.html'
+    end
+
+    # Count of pcaps by status.
+    get '/pcaps/1/status' do
+        db = settings.db
+        content_type :json
+        result = db.view 'pcaps/by_status', :group => true
+        return as_object(result).to_json
+    end
+
+    # High level statistics.
+    get '/pcaps/1/statistics' do
+        db = settings.db
+        content_type :json
+
+        result = db.view 'pcaps/statistics', :group => true
+
+        as_object(result).to_json
+    end
+
+    VIEWS = {
+        'date'      => 'pcaps/by_created_at',
+        'path'      => 'pcaps/by_path',
+        'status'    => 'pcaps/by_status',
+        'service'   => 'pcaps/by_service',
+        'keyword'   => 'pcaps/by_keyword',
+        'filename'  => 'pcaps/by_filename',
+        'directory' => 'pcaps/by_directory',
+    }
+
+    # List/query pcaps by CouchDB view.
+    get '/pcaps/1/list' do
+        content_type :json
+        db = settings.db
+        query = params.dup
+        by = query.delete('by') 
+        by ||= 'date'
+        view = VIEWS[by]
+        query.delete 'callback'
+        ['startkey', 'endkey', 'key'].each do |key| 
+            if val = query[key]
+                # JSON parser doesn't work unless value is an array or
+                # hash. (I.e. this fails: JSON.parse(1.to_json)
+                # So enclosing the value in a top level array.
+                parsed = JSON.parse("[#{val}]").pop
+                query[key] = parsed
+            end
+        end
+
+        db.view(view, query).to_json
+    end
+
+    # Returns doc for pcap.
+    get '/pcaps/1/about/:id' do
+        content_type :json
+        settings.db.get(params[:id]).to_json
+    end
+
+    # Deletes document if it has a failed status.
+    get '/pcaps/1/remove/:id' do
+        content_type :json
+        id = params[:id]
+        doc = settings.db.get(params[:id])
+        if doc and doc['status'] == 'failed'
+            settings.scanner.remove_doc doc
+            {'ok' => true}.to_json
+        else
+            {'error' => true, 'reason' => 'status is not failed'}.to_json
+        end
+    end
+
+    # Explore pcap on pcapr.
+    get '/pcaps/1/pcap/:id' do
+        id = params[:id]
+        doc = settings.db.get(params[:id])
+        if doc and doc['index']
+            version = doc['index']['about']['version']
+            location = "http://www.pcapr.net/xtractr/explore?version=#{version}&" \
+            "url=http://#{request.host_with_port}#{request.path}" 
+            redirect location
+        else
+            return {"error" => true , "message" => "not found"}.to_json
+        end
+    end
+
+    # Forward requests (GET) to xtractr browser instance.
+    get '/pcaps/1/pcap/:id/*' do
+        id  = params[:id]
+        path = params[:splat][0]
+        url = "#{path}?#{request.query_string}"
+        doc = settings.db.get(params[:id])
+        status, headers, body =  settings.xtractr.get(doc['filename'], url)
+        headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
+        return status, headers, body
+    end
+
+    # Forward requests (POST) to xtractr browser instance.
+    post '/pcaps/1/pcap/:id/*' do
+        id  = params[:id]
+        path = params[:splat][0]
+        url = "#{path}?#{request.query_string}"
+        doc = settings.db.get(params[:id])
+        status, headers, body =  settings.xtractr.post(doc['filename'], url, request.body.read)
+        headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
+        return status, headers, body
+    end
+
+    # Download archive that includes dissected and normalized pcap.
+    get '/pcaps/1/export_to_par/:id' do
+        content_type :par
+        id = params[:id]
+        doc = settings.db.get(params[:id])
+        raise "not found" unless doc
+        path = settings.scanner.pcap_path(doc)
+        io = Mu::Scenario::Pcap.export_to_par path
+        filename = File.basename(doc['filename']).gsub(/[.][^.]*$/, '.par')
+        return 200, {'Content-Disposition' => "filename=#{filename}" }, io
+    end
+
+    # Record any exceptions in main logfile.
+    after '/pcaps/*' do
+        if err=request.env['sinatra.error']
+            PcaprLocal::Logger.error "UI got an error: #{err.message}\n#{err.backtrace.join("\n")}"
+        end
+    end
+end
+end

data/lib/pcapr_local/www/home/index.html

@@ -0,0 +1,138 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <link rel="shortcut icon" href="/static/image/favicon.ico"/>
+    <title>pcapr.Local - A place to organize your packet captures</title>
+    
+    <link rel="stylesheet" type="text/css" href="/static/style/page.css" media="all"/>
+    <link rel="stylesheet" type="text/css" href="/static/style/c3p0.css" media="all"/>
+    <link rel="stylesheet" type="text/css" href="/static/style/jquery.suggest.css" media="all"/>
+    <link rel="stylesheet" type="text/css" href="/static/style/tipsy.css" media="all"/>
+    
+    <script type="text/javascript" src="/static/script/json2.js">
+	</script>
+	
+    <script type="text/javascript" src="/static/script/jquery/jquery-1.4.2.min.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery-ui.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.ui.core.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.ui.widget.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.ui.sortable.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.ui.slider.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.form.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.menu.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.suggest.js">
+	</script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.flot.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.flot.stack.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.flot.selection.js">
+    </script>
+    <script type="text/javascript" src="/static/script/jquery/jquery.jsonp.min.js">
+    </script>
+    <script type="text/javascript" src="/static/script/tipsy/jquery.tipsy.js">
+    </script>
+    
+    <script type="text/javascript" src="/static/script/sammy/sammy.js">
+    </script>
+    <script type="text/javascript" src="/static/script/sammy/plugins/sammy.template.js">
+    </script>
+    <script type="text/javascript" src="/static/script/sammy/plugins/sammy.cache.js">
+    </script>
+	
+    <script type="text/javascript" src="/static/script/closet/closet.api.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.util.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.quantity.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.render.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.folders.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.mr.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.options.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.report.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.reports.visualize.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.reports.overview.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.reports.tcp.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.reports.http.js">
+    </script>
+    <script type="text/javascript" src="/static/script/closet/closet.reports.sip.js">
+    </script>
+    
+    
+    <script type="text/javascript" src="/static/script/closet/async.js">
+    </script>
+    
+   
+    <script type="text/javascript">
+        (function($) {
+            $(function() { closet.load(); });
+        }(jQuery));
+    </script>
+</head>
+<body id="page_bg" class="f-default">
+<div id="container">
+<div id="mainbody">
+<div class="wrapper">
+<div class="content beta">
+    <div style="float:left;font-size:large;margin-top:6px;margin-left:5px;padding-right:40px">
+        <a style="text-decoration:none;outline:none" href="http://www.mudynamics.com">
+            <img class="icon" src="/static/image/favicon.png"/>
+        </a>
+        <span style="font-weight:bold;letter-spacing:0.1em">pcapr.Local</span>
+    </div>
+        
+    <div class="browse">
+        <div class="floater" style="float:right">
+            <table style="display:inline">
+                <tr>
+                    <td><img class="icon" src="/static/image/16x16/Search.png"/></td>
+                    <td><input id="search" type="text"/>&nbsp;</td>
+                    <td>
+                        <div class="statistics" style="float:right"></div>
+                    </td>
+                </tr>
+            </table>
+        </div>
+        <ul class="tab">
+            <li class="active">
+                <a id="browse" href="#/browse"><span>Browse</span></a>
+            </li>
+            <li>
+                <a id="folders" href="#/folders"><span>Folders</span></a>
+            </li>
+            <li>
+                <a id="services" href="#/services"><span>Services</span></a>
+            </li>
+        </ul>
+        <div class="browse-tab">
+        </div>
+    </div>
+    
+    <div class="footer">
+    </div>
+</div>
+</div>
+</div>
+</div>
+</body>
+</html>