dorothy2 0.0.3 → 1.0.0

data/lib/mu/xtractr/packets.rb

@@ -0,0 +1,122 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Packets
+# Packets is an iterator on a collection of packets determined by the search
+# query. This class can also be used to report on the collection of packets
+# in addition to slicing those packets into a new pcap.
+#
+#  xtractr.packets('http.request.method:GET pkt.src:192.168.1.1').save('foo.pcap')
+#
+# <b>Find the top URL's in HTTP packets</b>
+#
+#  xtractr.packets('pkt.service:HTTP').count('http.request.method')
+#
+# <b>Find the packet size distribution for DNS packets</b>
+#
+#  xtractr.packets('pkt.service:DNS').count('pkt.length')
+class Packets
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    MAX_PAGE_SIZE = 100 # :nodoc:
+    
+    def initialize xtractr, opts # :nodoc:
+        @xtractr = xtractr
+        @opts = opts
+        @opts[:q] ||= '*'
+    end
+    
+    def q # :nodoc:
+        @opts[:q]
+    end
+    
+    # Iterate over each packet that matches the search criteria. It's always
+    # better to use this with a fine-grained query instead of packets.to_a
+    # because it's going to try and load <em>all</em> packets from the index.
+    #  xtractr.packets("pkt.src:192.168.1.1").each do |pkt|
+    #      ...
+    #  end
+    def each_packet() # :yields: packet
+        _opts = @opts.dup
+        _opts[:start] ||= 1
+        _opts[:limit] ||= MAX_PAGE_SIZE
+        
+        while true
+            result = xtractr.json "api/packets", _opts
+            rows = result['rows']            
+            break if rows.empty?
+                        
+            rows[0, MAX_PAGE_SIZE-1].each do |row| 
+                packet = Packet.new xtractr, row 
+                yield packet
+            end
+            
+            break if rows.size < MAX_PAGE_SIZE
+            _opts[:start] = rows[MAX_PAGE_SIZE-1]['id']
+        end        
+        return self
+    end
+    
+    # Fetch the first packet that matched the query. Mostly used for unit
+    # testing.
+    def first
+        result = xtractr.json "api/packets", :start => 1, :limit => 1, :q => q
+        rows = result['rows']
+        rows.empty? ? nil : Packet.new(xtractr, rows[0])
+    end
+    
+    # Count the unique values of the specified field amongst all the packets
+    # that matched the query.
+    #  xtractr.packets('mozilla').count('http.request.uri')
+    def count field
+        Views.count xtractr, field, '/api/packets/report', @opts
+    end
+    
+    # Return a list of Field::Value objects for the specified field, sorted
+    # by their frequency. This is a convenience method used in method chaining.
+    #  xtractr.packets('index.html').values('http.request.uri')
+    def values field
+        count(field).map { |c| c.object }
+    end
+    
+    # Sum the numeric values of vfield, keyed by the unique values of
+    # kfield.
+    #  xtractr.packets('mozilla').sum('http.request.uri', 'pkt.length')
+    def sum kfield, vfield
+        Views.sum xtractr, kfield, vfield, '/api/packets/report', @opts
+    end
+    
+    # Stich together a pcap made up of all packets that matched the query
+    # and save it to the filename.
+    #  xtractr.packets('pkt.service:DNS pkt.length:>64').save('foo.pcap')
+    def save filename
+        open(filename, "w") do |ios|
+            pcap = xtractr.get "api/packets/slice", :q => @opts[:q]
+            ios.write pcap
+        end
+        return self
+    end
+    
+    def inspect # :nodoc:
+        "#<packets:#{@opts[:q]}>"
+    end
+        
+    alias_method :each, :each_packet
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/service.rb

@@ -0,0 +1,77 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Service
+# The Service class represents the Wireshark-assigned protocol that's somewhat 
+# higher level than the IP protocol. You can get a list of all the unique
+# services in your index through the xtractr's methods:
+#
+#  xtractr.services(/http/).each { |service| ... }
+#  xtractr.services.each { |service| ... }
+#
+# Some services like HTTP have counterparts in both flows and packets, while
+# others like ARP (all non-IP, layer2 services) are only available in packets.
+class Service
+    attr_reader :xtractr # :nodoc:
+    
+    # Return the name of the service
+    attr_reader :name
+    
+    def initialize xtractr, name # :nodoc:
+        @xtractr = xtractr
+        @name = name
+    end
+    
+    # Get a unique list of clients for this service
+    #  xtractr.service('http').clients
+    def clients q=nil
+        _q = "flow.service:\"#{name}\""
+        _q << " #{q}" if q
+        Flows.new(xtractr, :q => _q).count('flow.src')
+    end
+    
+    # Get a unique list of servers for this service
+    #  xtractr.service('http').servers
+    def servers q=nil
+        _q = "flow.service:\"#{name}\""
+        _q << " #{q}" if q
+        Flows.new(xtractr, :q => _q).count('flow.dst')
+    end
+    
+    # Return an iterator that can yield all packets that have this service and
+    # matches the query
+    #  xtractr.service("DNS").packets("mu").each { |pkt| ... }
+    def packets q=nil
+        _q = "pkt.service:\"#{name}\""
+        _q << " #{q}" if q
+        return Packets.new(xtractr, :q => _q)
+    end
+    
+    # Return an iterator that can yield all flows that have this service and
+    # matches the query
+    #  xtractr.service("DNS").flows("AAAA").each { |flow| ... }
+    def flows q=nil
+        _q = "flow.service:\"#{name}\""
+        _q << " #{q}" if q
+        return Flows.new(xtractr, :q => _q)
+    end
+    
+    def inspect # :nodoc:
+        "#<service:#{name}>"
+    end
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/stream/http.rb

@@ -0,0 +1,103 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'stringio'
+require 'zlib'
+
+module Mu
+class Xtractr
+class Stream
+class HTTP < Processor # :nodoc:
+    # Check to see if this is a HTTP stream.
+    def self.matches? stream
+        not stream.messages.select { |message| message.bytes =~ /HTTP\/1\./ }.empty?
+    end
+    
+    # Pull out the response body from the messages in the stream and convert
+    # them into contents on the stream.
+    def self.extract stream
+        stream.messages.each { |message| process stream, message }
+    end
+    
+    def self.process stream, message
+        content = Content.new message
+        chunked = false
+        length = nil
+        ios = StringIO.new message.bytes
+        while true
+            line = ios.readline.chomp rescue nil
+            break if not line or line.empty?
+            
+            case line
+            when /Content-Length:\s*(\d+)/i
+                length = $1.to_i
+            when /Content-Type:\s*(.*)/i
+                content.type = $1
+            when /Content-Encoding:\s*(.*)/i
+                content.encoding = $1
+            when /Transfer-Encoding:\s*chunked/i
+                chunked = true
+            end
+        end
+        
+        # Read the content
+        bytes = ios.read(length)
+        return if not bytes or bytes.empty?
+                        
+        # Handle chunked encoding, if necessary
+        bytes = dechunk(bytes) if chunked
+        
+        # And then decompress, if necessary
+        if ['gzip','deflate'].member? content.encoding
+            bytes = decompress(bytes, content.encoding)
+        end
+        
+        if bytes
+            content.body = bytes
+            stream.contents << content
+        end
+    end
+    
+    def self.dechunk text
+        ios = StringIO.new text
+        body = ''
+        while true
+            line = ios.readline rescue nil
+            break if not line or line.empty?
+            
+            chunksz = line.to_i(16)
+            break if chunksz.zero?
+            
+            body << ios.read(chunksz)
+        end
+        return body
+    end
+    
+    def self.decompress text, method
+        if method == 'gzip'
+            ios = StringIO.new text
+            reader = Zlib::GzipReader.new ios
+            begin
+                return reader.read
+            ensure
+                reader.close
+            end
+        elsif method == 'deflate'
+            return Zlib::Inflate.inflate(text)
+        end
+    end
+end
+end # Stream
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/stream.rb

@@ -0,0 +1,132 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Stream
+# Represents a logical TCP stream made of messages (potentially spanning
+# multiple packets across multiple pcaps). A given message in the stream is
+# potentially stitched together from multiple packets. Streams form the basis
+# of doing content-analysis with xtractr. xtractr does all the work of
+# reassembling the packets and pulling out the appropriate payload from
+# each of the packets.
+#
+#  xtractr.flows('flow.service:http').first.stream.find do |message|
+#      m =~ /xml/
+#  end
+#
+# Each stream also has a set of content processors that are invoked at 
+# creation time which pull out attachments, images, etc from the reassembled
+# stream.
+#
+#  xtractr.flows('flow.service:http').first.stream.contents.each do |content|
+#      content.save if content.type == 'image/jpeg'
+#  end
+class Stream
+    include Enumerable
+    
+    # A list of stream processors that can pull out content from messages
+    class Processor # :nodoc:
+        def self.inherited klass
+            @processors ||= [] << klass
+        end
+        
+        def self.processors
+            @processors ||= []
+        end
+    end
+    
+    attr_reader :xtractr # :nodoc:
+    
+    # Return the flow that this stream represents.
+    attr_reader :flow
+    
+    # Return a list of Messages in this stream.
+    attr_reader :messages
+    
+    # Return a list of extracted content from the messages.
+    attr_reader :contents
+    
+    # = Message
+    # Represents a single logical TCP message that has been potentially
+    # reassembled from across multiple packets spanning multiple pcaps. Each
+    # message contains the stream to which it belongs in addition to whether
+    # this message was sent from the client or the server.
+    class Message
+        # Returns the stream to which this message belongs to.
+        attr_reader :stream
+        
+        # Returns the index within the stream
+        attr_reader :index
+        
+        # Returns the direction of the message (request/response).
+        attr_reader :dir
+        
+        # Returns the actual bytes of the message.
+        attr_reader :bytes
+        
+        def initialize stream, index, dir, bytes # :nodoc:
+            @stream = stream
+            @index  = index
+            @dir    = dir
+            @bytes  = bytes
+        end
+        
+        def inspect # :nodoc:
+            preview = bytes[0..32]
+            preview << "..." if bytes.size > 32
+            return "#<message:#{index} flow-#{stream.flow.id} #{preview.inspect}>"
+        end
+    end
+    
+    def initialize xtractr, flow, json # :nodoc:
+        @xtractr  = xtractr
+        @flow     = flow
+        @messages = []
+        @contents = []
+        
+        json['packets'].each do |pkt|
+            bytes = (pkt['b'] || []).map { |b| b.chr }.join('')
+            if messages.empty? or messages[-1].dir != pkt['d']
+                messages << Message.new(self, messages.size, pkt['d'], '')
+            end
+            messages[-1].bytes << bytes
+        end
+        
+        # Run the stream/messages through each registered processor to pull
+        # out attachments, files, etc
+        Processor.processors.each do |processor|
+            if processor.matches? self
+                processor.extract self
+                break
+            end
+        end
+    end
+    
+    # Iterate over each message in this stream
+    def each_message &blk
+        messages.each(&blk)
+        return self
+    end
+    
+    def inspect # :nodoc:
+        return "#<stream:#{flow.id} ##{messages.size} messages>"
+    end
+    
+    alias_method :each, :each_message
+end
+end # Xtractr
+end # Mu
+
+require 'mu/xtractr/stream/http'

data/lib/mu/xtractr/term.rb

@@ -0,0 +1,73 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Term
+# A term represents a tokenized value of a field that also contains the
+# frequency of occurence across all of the packets in a given index. Terms
+# are useful to get quick snapshots of the index as well as in trend analysis.
+# For example the following shows the top HTTP request methods as well as the
+# frequency of those methods across all of the packets.
+#
+#  xtractr.field('http.request.method').terms.each do |term|
+#    p [ term.value, term.frequency ]
+#  end
+class Term
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    # Return the field containing this term.
+    attr_reader :field
+    
+    # Return the value of this term.
+    attr_reader :value
+    
+    # Return the (packet) frequency of this term.
+    attr_reader :frequency
+    
+    def initialize field, json # :nodoc:
+        @field     = field
+        @xtractr   = field.xtractr
+        @value     = json['key']
+        @frequency = json['value']
+    end
+    
+    # Fetch each packet from the index that has this term in this field. If
+    # the optional q is specified, the search query is AND'd with the term's
+    # own search query
+    #  field.term('mozilla').each { |pkt| ... }
+    def each_packet(q=nil, &blk) # :yields: packet
+        _q = "#{field.name}:#{value}"
+        _q << " #{q}" if q
+        return Packets.new(xtractr, :q => _q).each(&blk)
+    end
+    
+    # Return an instance of Packets that serves as an iterator for all packets
+    # containing this term.
+    def packets q=nil
+        _q = "#{field.name}:#{value}"
+        _q << " #{q}" if q
+        return Packets.new(xtractr, :q => _q)
+    end
+    
+    def inspect # :nodoc:
+        "#<term:#{field.name} #{value} #{frequency}>"
+    end
+
+    alias_method :each, :each_packet
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/test/stream/tc_http.rb

@@ -0,0 +1,53 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'mu/xtractr'
+require 'test/unit'
+
+module Mu
+class Xtractr
+class Stream
+class HTTP
+class Test < Test::Unit::TestCase
+    attr_reader :xtractr
+    attr_reader :flow
+    
+    def setup
+        @xtractr = Xtractr.new
+    end
+    
+    def test_extract
+        stream = xtractr.packets('http.content.type:gif').first.flow.stream
+        assert_equal(1, stream.contents.size)
+        content = stream.contents.first
+        assert_nothing_raised { content.inspect }
+        assert_equal('image/gif', content.type)
+        assert_nil(content.encoding)
+        assert_equal('content.4.1', content.name)
+        assert_equal(content.message.__id__, stream.messages[1].__id__)
+        
+        stream = xtractr.packets('http.content.type:xml').first.flow.stream
+        assert_equal(1, stream.contents.size)
+        content = stream.contents.first
+        assert_nothing_raised { content.inspect }
+        assert_equal('text/xml', content.type)
+        assert_equal('gzip', content.encoding)
+        assert_equal('content.2.1', content.name)
+        assert_equal(content.message.__id__, stream.messages[1].__id__)
+    end    
+end
+end # HTTP
+end # Stream
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/test/tc_field.rb

@@ -0,0 +1,140 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'mu/xtractr'
+require 'test/unit'
+
+module Mu
+class Xtractr
+class Field
+class Test < Test::Unit::TestCase
+    attr_reader :xtractr
+    
+    def setup
+        @xtractr = Xtractr.new
+    end
+    
+    def test_Field
+        assert(Field.ancestors.include?(Enumerable), "Field doesn't mixin Enumerable")
+    end
+    
+    def test_each
+        field = xtractr.field('pkt.src')
+        terms = field.terms
+        assert_equal(11, terms.size)
+        terms.each { |t| assert_kind_of(Term, t) }
+        
+        val = field.each_term { |t| assert_kind_of(Term, t) }
+        assert_equal(field, val)
+    end
+    
+    def test_terms
+        field = xtractr.field('dns.qry.name')
+        terms = field.terms
+        assert_equal(12, terms.size)
+        assert_equal(true, terms.first.frequency > terms.last.frequency)
+        
+        field.terms(/itunes/).each do |term|
+            assert_match(/itunes/, term.value)
+        end
+    end
+        
+    def test_term
+        field = xtractr.field('dns.qry.name')
+        
+        [ :[], :term ].each do |method|
+            term = field.send method, 'itunes'
+            assert_kind_of(Term, term)
+            assert_equal('itunes', term.value)
+            assert_equal(3, term.frequency)
+        end
+        
+        assert_raise(ArgumentError) { field['foo'] }
+    end
+    
+    def test_count
+        field = xtractr.field('dns.qry.name')
+        counts = field.count
+        assert_equal(7, counts.size)
+        counts.each { |v| assert_kind_of(Views::Count, v) }
+        
+        counts = field.count('flow.dst:4.2.2.1')
+        assert_equal(4, counts.size)
+        counts.each { |v| assert_kind_of(Views::Count, v) }
+    end
+    
+    def test_values
+        field = xtractr.field('dns.qry.name')
+        values = field.values
+        assert_equal(7, values.size)
+        values.each { |v| assert_kind_of(Field::Value, v) }
+        
+        values = field.values('flow.dst:4.2.2.1')
+        assert_equal(4, values.size)
+        values.each { |v| assert_kind_of(Field::Value, v) }
+    end
+    
+    def test_inspect
+        field = xtractr.field('dns.qry.name')
+        assert_nothing_raised { field.inspect }
+    end
+    
+    class Value
+    class Test < ::Test::Unit::TestCase
+        attr_reader :xtractr
+        attr_reader :value
+        
+        def setup
+            @xtractr = Xtractr.new
+            @value = xtractr.field('dns.qry.name').values.first
+        end
+        
+        def test_q
+            assert_equal('dns.qry.name:"ax.search.itunes.apple.com"', value.q)
+        end
+        
+        def test_packets
+            packets = value.packets
+            assert_kind_of(Packets, packets)
+            assert_equal('dns.qry.name:"ax.search.itunes.apple.com"', packets.q)
+        end
+        
+        def test_each_packet
+            v = value.each_packet { |pkt| assert_kind_of(Packet, pkt) }
+            assert_equal(value, v)
+        end
+        
+        def test_count
+            value.count('pkt.src').each do |c|
+                assert_kind_of(Views::Count, c)
+                assert_equal('pkt.src', c.field.name)
+            end
+        end
+        
+        def test_sum
+            value.sum('pkt.src', 'pkt.length').each do |s|
+                assert_kind_of(Views::Sum, s)
+                assert_equal('pkt.src', s.field.name)
+            end
+        end
+        
+        def test_inspect
+            assert_nothing_raised { value.inspect }
+        end
+    end
+    end
+end
+end # Field
+end # Xtractr
+end # Mu