dorothy2 0.0.3 → 1.0.0

data/lib/mu/xtractr/field.rb

@@ -0,0 +1,178 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Field
+# Field represents a named packet field in the <b>xtractr</b> index. Each field 
+# contains tokenized terms along with the frequency at which they occur. A
+# field is a queryable object and can be used to iterate over flows or packets
+# that match the Field::Value. See Flow or Packet for more information on the
+# fields that are stored in the index.
+#
+#  xtractr.field('http.request.uri').each_packet { |packet ... }
+#
+# The fields are useful for quick analysis on top trending terms across the
+# entire index. Here are the top HTTP fields that have NOP slides in them:
+#
+#  xtractr.fields(/^http/).select do |field|
+#     not field.terms(/AAAAAA/i).empty?
+#  end
+class Field
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    # The name of the field.
+    attr_reader :name
+    
+    # = Field::Value
+    # Field::Value represents an instance of a field with a concrete value that 
+    # can further used for fine grained searches.
+    class Value
+        attr_reader :xtractr # :nodoc:
+        
+        # Return the field object.
+        attr_reader :field
+        
+        # Return the value of the field object.
+        attr_reader :value
+        
+        def initialize xtractr, json # :nodoc:
+            @xtractr = xtractr
+            @field = Field.new(xtractr, json['key'])
+            @value = json['value']
+        end
+        
+        def q # :nodoc:
+            "#{field.name}:\"#{value}\""
+        end
+        
+        # Fetch the list of packets that contain this Field::Value. If the
+        # optional query is given, it's AND'd to the query that matches this
+        # Field::Value.
+        #  value.packets.each { |pkt| ... }
+        #  value.packets('dns.qry.name:apple').each { |pkt| ... }
+        def packets _q=nil
+            q2 = q
+            q2 << " #{_q}" if _q
+            Packets.new xtractr, :q => q2
+        end
+        
+        # Iterate over each packet that contains this field value. This is
+        # a convenience function used primiarily in method chaining.
+        def each_packet(_q=nil, &blk) # :yields: packet
+            packets(_q).each(&blk)
+            return self
+        end
+        
+        # Count the unique values of the specified field amongst all the packets
+        # that matched the query.
+        #  value.count('http.request.method')
+        def count _field
+            which = field.name =~ /^flow\./ ? 'flows' : 'packets'
+            Views.count xtractr, _field, "/api/#{which}/report", :q => q
+        end
+        
+        # Sum the unique numeric values of vfield, keyed by the unique values of
+        # kfield.
+        #   value.sum('flow.src', 'flow.bytes')
+        def sum kfield, vfield
+            which = field.name =~ /^flow\./ ? 'flows' : 'packets'
+            Views.sum xtractr, kfield, vfield, "/api/#{which}/report", :q => q
+        end
+        
+        def inspect # :nodoc:
+            "#<value:#{field.name} #{value}>"
+        end
+    end
+    
+    def initialize xtractr, name # :nodoc:
+        @xtractr = xtractr
+        @name = name
+    end
+    
+    # Fetch the terms and their packet frequencies (in packets) for this field.
+    # If the optional start term is given, then the term enumeration starts
+    # from the specified term.
+    #  field.each { |term| ... }
+    #  field.each('mozilla') { |term| ... }
+    def each_term(start='') # :yields: term
+        opts = {}
+        opts[:start] = start
+        opts[:limit] = 101
+        
+        while true
+            result = xtractr.json "api/field/#{name}/terms", opts
+            rows = result['rows']            
+            break if rows.empty?
+            
+            rows[0, 100].each do |row| 
+                term = Term.new self, row 
+                yield term
+            end
+            
+            break if rows.size < 101
+            opts[:start] = rows[100]['key']
+        end
+        
+        return self
+    end
+    
+    # Fetch the list of <em>all</em> the unique terms for this field, sorted by the
+    # frequency of occurence in the packets. This can be used for some quick
+    # trend analysis to see which term of a given field appears most amongst
+    # all packets in the index. Here's an example to print out the top 10 terms
+    # of <em>http.request.uri</em>.
+    #  p xtractr.field('http.request.uri').terms[0..10]
+    def terms regex=nil
+        regex = Regexp.new(regex, Regexp::IGNORECASE) if regex.is_a? String
+        t = regex ? entries.select { |name| name =~ regex } : entries
+        t.sort { |a, b| b.frequency <=> a.frequency }
+    end
+    
+    # Find the term for this field which has the name and the packet frequency.
+    #  field.term 'mozilla'
+    def [] which
+        result = xtractr.json "api/field/#{name}/terms", :start => which, :limit => 1
+        rows = result['rows']
+        if rows.empty? || rows[0]['key'] != which
+            raise ArgumentError, "Unknown term #{which} for field #{name}"
+        end
+        return Term.new(self, rows[0])
+    end
+    
+    # Find out all the unique values of this field with an optional query.
+    #  xtractr.field('http.user.agent').count('flow.src:192.168.1.1')
+    def count q='*'
+        Views.count xtractr, self, "api/flows/report", :q => q
+    end
+    
+    # Return a list of Field::Value objects for this field, sorted by their 
+    # frequency. This is a convenience method to use the resulting Field::Value
+    # objects in method chaining.
+    #  xtractr.field('http.user.agent').values.first.packets.slice('foo.pcap')
+    def values q='*'
+        count(q).map { |c| c.object }
+    end
+    
+    def inspect # :nodoc:
+        "#<field:#{name}>"
+    end
+    
+    alias_method :each, :each_term
+    alias_method :term, :[]
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/flow.rb

@@ -0,0 +1,162 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Flow
+# Flow represents a single conversation between two hosts. Depending on whether
+# it's IP, UDP or TCP, xtractr uses different information from the IP header
+# of the packets (src/dst addresses and ports) to logically group them
+# together. Each flow also has the duration (difference in the timestamp between
+# the first and last packet in the conversation), the total bytes that were
+# exchanged as well as the logical messages that were exchanged. For example:
+#
+# <b>Identify hosts that performed TCP port scans</b>
+#
+#  xtractr.flows('flow.proto:6 flow.cmsgs:0 flow.smsgs:0').count('flow.src')
+#
+# <b>Identify DNS queries with no response (possibly timed out)</b>
+#
+#  xtractr.flows('flow.service:DNS flow.smsgs:0').count('dns.qry.name')
+class Flow
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    # The unique ID of the flow.
+    attr_reader :id
+    
+    # The timestamp of the flow, determined by the first packet in the flow.
+    attr_reader :time
+    
+    # The duration of the flow, determined by the first and last packet in the flow.
+    attr_reader :duration
+    
+    # The source host of the flow.
+    attr_reader :src
+    
+    # The destination host of the flow.
+    attr_reader :dst
+    
+    # The IP protocol of the flow.
+    attr_reader :proto
+    
+    # The source port of the flow (if applicable).
+    attr_reader :sport
+    
+    # The destination port of the flow (if applicable).
+    attr_reader :dport
+    
+    # The service of the flow (like DNS or HTTP).
+    attr_reader :service
+    
+    # The title of the flow.
+    attr_reader :title
+    
+    # The total ##packets in the flow.
+    attr_reader :packets
+    
+    # The total ##bytes (request and response) in the flow.
+    attr_reader :bytes
+    
+    # The logical client messages (payloads) in the flow.
+    attr_reader :cmsgs
+    
+    # The logical server messages (payloads) in the flow.
+    attr_reader :smsgs
+    
+    def initialize xtractr, json # :nodoc:
+        @xtractr  = xtractr
+        @id       = json['id']
+        @time     = json['time']
+        @duration = json['duration']
+        @src      = Host.new xtractr, json['src']
+        @dst      = Host.new xtractr, json['dst']
+        @proto    = json['proto']
+        @sport    = json['sport']
+        @dport    = json['dport']
+        @service  = Service.new xtractr, json['service']
+        @title    = json['title']
+        @packets  = json['packets']
+        @bytes    = json['bytes']
+        @cmsgs    = json['cmsgs']
+        @smsgs    = json['smsgs']
+        @first_id = json['first']
+        @last_id  = json['last']
+        @iterator = Packets.new(xtractr, :q => "pkt.flow:#{id}")
+    end
+    
+    # Return the first packet for this flow. Together the first and last
+    # packets make up the span of the flow. Read this
+    # blog[http://labs.mudynamics.com/2010/09/30/visualizing-application-flows-with-xtractr/]
+    # to see how these spans enable flow visualization.
+    #  xtractr.flow(1).first.bytes
+    def first
+        @first ||= xtractr.packet @first_id
+    end
+    
+    # Return the last packet for this flow. Together the first and last
+    # packets make up the span of the flow. Read this
+    # blog[http://labs.mudynamics.com/2010/09/30/visualizing-application-flows-with-xtractr/]
+    # to see how these spans enable flow visualization.
+    #  xtractr.flow(2).last.bytes
+    def last
+        @last ||= xtractr.packet @last_id
+    end
+    
+    # Iterate over each packet in this flow.
+    #  flow.each { |pkt| ... }
+    def each_packet(&blk) # :yields: packet
+        @iterator.each(&blk)
+        return self
+    end
+    
+    # Reassemble the TCP stream for this flow (assuming it's a TCP flow) and
+    # return the stream. This is the basis for doing content extraction from
+    # packets even if the packets span multiple pcaps.
+    #  xtractr.service('HTTP').flows.first.stream
+    def stream
+        result = xtractr.json "api/flow/#{id}/stream"
+        return Stream.new(xtractr, self, result)
+    end
+    
+    # A convenience method to fetch the stream for this flow, extract the
+    # content and then return an array of contents.
+    #  xtractr.flows('flow.service:HTTP favicon.ico').each do |flow|
+    #      flow.contents.each { |c| c.save }
+    #  end
+    def contents
+        stream.contents
+    end
+    
+    # Stich together a pcap made up of all packets containing this flow and
+    # save it to the filename. It's possible for the packets to span multiple
+    # pcaps, but xtractr makes it seamless.
+    #  flow.save("foo.pcap")
+    def save filename
+        open(filename, "w") do |ios|
+            pcap = xtractr.get "api/packets/slice", :q => "pkt.flow:#{id}"
+            ios.write pcap
+        end
+        return self
+    end
+    
+    def inspect # :nodoc:
+        "#<flow:#{id} #{service.name} #{src.address}:#{sport} > #{dst.address}:#{dport} #{title}"
+    end
+    
+    alias_method :each, :each_packet
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/flows.rb

@@ -0,0 +1,118 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# Flows is an iterator for the flows in the index, based on a given query.
+# The default query for this iterator is '*', implying that it will iterate
+# over <em>all</em> the flows in the index.
+class Flows
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    MAX_PAGE_SIZE = 100 # :nodoc:
+    
+    def initialize xtractr, opts # :nodoc:
+        @xtractr = xtractr
+        @opts = opts.dup
+        @opts[:q] ||= '*'
+    end
+    
+    def q # :nodoc:
+        @opts[:q]
+    end
+    
+    # Iterate over each flow that matches the search criteria. It's always
+    # better to use this with a fine-grained query instead of flows.to_a
+    # because it's going to try and load *all* flows from the index.
+    #  xtractr.flows("flow.src:192.168.1.1").each { |flow| ... }
+    def each_flow() # :yields: flow
+        _opts = @opts.dup
+        _opts[:start] ||= 1
+        _opts[:limit] ||= MAX_PAGE_SIZE
+        
+        while true
+            result = xtractr.json "api/flows", _opts
+            rows = result['rows']            
+            break if rows.empty?
+                        
+            rows[0, MAX_PAGE_SIZE-1].each do |row| 
+                flow = Flow.new xtractr, row 
+                yield flow
+            end
+            
+            break if rows.size < MAX_PAGE_SIZE
+            _opts[:start] = rows[MAX_PAGE_SIZE-1]['id']
+        end
+        return self
+    end
+    
+    # Fetch the first flow that matched the query. This is mainly used for
+    # unit testing, but useful within IRB to experiment with method chaining.
+    #  flows.first.save("1.pcap")
+    def first
+        result = xtractr.json "api/flows", :start => 1, :limit => 1, :q => q
+        rows = result['rows']
+        rows.empty? ? nil : Flow.new(xtractr, rows[0])
+    end
+    
+    # Count the unique values of the specified field amongst all the flows
+    # that matched the query.
+    #  xtractr.flows('index.html').count('http.request.uri')
+    def count field
+        Views.count xtractr, field, '/api/flows/report', @opts
+    end
+    
+    # Return a list of Field::Value objects for the specified field, sorted
+    # by their frequency. This is a convenience method used in method chaining.
+    #  xtractr.flows('index.html').values('http.request.uri')
+    def values field
+        count(field).map { |c| c.object }
+    end
+    
+    # Sum the numeric values of vfield, keyed by the unique values of
+    # kfield.
+    #  xtractr.flows('index.html').sum('http.request.uri', 'flow.bytes')
+    def sum kfield, vfield
+        Views.sum xtractr, kfield, vfield, '/api/flows/report', @opts
+    end
+    
+    # Save all the packets for this collection of flows into a pcap. It's
+    # possible that the packets for the flows might span multiple indexed
+    # pcaps.
+    #  xtractr.flows('flow.service:DNS AAAA').save('dns.pcap')
+    def save filename
+        flow_ids = []
+        each_flow do |flow| 
+            flow_ids << flow.id.to_s
+            break if flow_ids.size >= 1024
+        end
+        
+        _q = "pkt.flow:(" << flow_ids.join('||') << ')'
+        open(filename, "w") do |ios|
+            pcap = xtractr.get "api/packets/slice", :q => _q
+            ios.write pcap
+        end
+        return self
+    end
+    
+    def inspect # :nodoc:
+        "#<flows:#{@opts[:q]}>"
+    end
+        
+    alias_method :each, :each_flow
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/host.rb

@@ -0,0 +1,87 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Host
+# Host is a generic entity representing MAC, IPv4 and IPv6 addresses. You can
+# get a list of all the unique hosts in the index using the root xtractr instance.
+#  xtractr.hosts
+class Host
+    attr_reader :xtractr # :nodoc:
+    
+    # Returns the address of this host.
+    attr_reader :address
+    
+    def initialize xtractr, address # :nodoc:
+        @xtractr = xtractr
+        @address = address
+    end
+    
+    # Use the host as a server and get a unique list of its clients.
+    #  xtractr.hosts(/192.168/).first.clients
+    def clients q=nil
+        _q = role2q :server, 'flow', q
+        Flows.new(xtractr, :q => _q).count('flow.src')
+    end
+    
+    # Use the host as a client and get a unique list its servers.
+    #  xtractr.hosts(/192.168/).first.servers
+    def servers q=nil
+        _q = role2q :client, 'flow', q
+        Flows.new(xtractr, :q => _q).count('flow.dst')
+    end
+    
+    # Get a unique list of the host's services. <em>role</em> can be one of
+    # :any, :client or :server to specify the role.
+    #  host.services :server
+    def services role =:any, q=nil
+        _q = role2q role, 'flow', q
+        Flows.new(xtractr, :q => _q).count('flow.service')
+    end
+    
+    # Return a flow iterator to iterate over the various flows that contain
+    # this host in the specified role.
+    #  host.flows :client
+    def flows role =:any, q=nil
+        _q = role2q role, 'flow', q
+        Flows.new(xtractr, :q => _q)
+    end
+    
+    # Return a packet iterator to iterate over the various packets that contain
+    # this host in the specified role.
+    #  host.packets :server
+    def packets role =:any, q=nil
+        _q = role2q role, 'pkt', q
+        Packets.new(xtractr, :q => _q)
+    end
+    
+    def inspect # :nodoc:
+        "#<host:#{address}>"
+    end
+    
+    private
+    def role2q role, forp, q=nil # :nodoc:
+        _q = case role
+                when :any:    "#{forp}.src|#{forp}.dst:\"#{address}\""
+                when :client: "#{forp}.src:\"#{address}\""
+                when :server: "#{forp}.dst:\"#{address}\""
+                else raise ArgumentError, "Unknown role #{role}"
+            end
+        _q << " #{q}" if q
+        return _q
+    end
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/packet.rb

@@ -0,0 +1,138 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# = Packet
+# A class that represents a single packet in the xtractr index. You can
+# iterate over all the packets in the index like this:
+#  xtractr.packets('blah').each { |pkt| ... }
+class Packet
+    include Enumerable
+    
+    attr_reader :xtractr # :nodoc:
+    
+    # The unique ID of the packet.
+    attr_reader :id
+    
+    # The file offset of the packet within the pcap.
+    attr_reader :offset
+    
+    # The length of the packet (the entire frame).
+    attr_reader :length
+    
+    # The relative timestamp of the packet.
+    attr_reader :time
+    
+    # The direction of the packet (if it belongs to a flow).
+    attr_reader :dir
+    
+    # The source host of the packet.
+    attr_reader :src
+    
+    # The destination host of the packet.
+    attr_reader :dst
+    
+    # The service of the packet.
+    attr_reader :service
+    
+    # The title of the packet.
+    attr_reader :title
+    
+    def initialize xtractr, json # :nodoc:
+        @xtractr = xtractr
+        @id      = json['id']
+        @offset  = json['offset']
+        @length  = json['length']
+        @pcap_id = json['pcap']
+        @flow_id = json['flow']
+        @time    = json['time']
+        @dir     = json['dir']
+        @src     = Host.new xtractr, json['src']
+        @dst     = Host.new xtractr, json['dst']
+        @service = Service.new xtractr, json['service']
+        @title   = json['title']
+    end
+    
+    # Returns the flow (if any) that this packet belongs to.
+    #  xtractr.packets('index.html').first.flow
+    def flow
+        return nil if @flow_id.zero?
+        @flow ||= xtractr.flow @flow_id
+    end
+        
+    # Fetch the actual packet data from the index. The return value is a
+    # String (that might contain null characters).
+    #  xtractr.packets('index.html').first.bytes
+    def bytes
+        result = xtractr.json "/api/packet/#{id}/bytes"
+        result['bytes'].map { |b| b.chr }.join('')
+    end
+    
+    # For UDP/TCP (both IPv4 and IPv6) packets, fetch just the layer4 payload. 
+    # Returns an empty string for all other types of packet.
+    #  xtractr.packets('http.request.method:GET').each do |pkt|
+    #      puts pkt.payload
+    #  end
+    def payload
+        result = xtractr.json "/api/packet/#{id}/bytes"
+        bytes = result['bytes']
+        l4size = result['l4size'] || 0
+        bytes[-l4size, l4size].map { |b| b.chr }.join('')
+    end
+    
+    # Iterate over each Field::Value in the packet. The various packet fields
+    # are only available if the indexing was done with <em>--mode forensics</em>.
+    #   packet.each('ip.ttl') { |fv| ... }
+    def each_field(regex=nil) # :yields: value
+        regex = Regexp.new(regex) if regex.is_a? String
+        result = xtractr.json "/api/packet/#{id}/fields"
+        rows = result['rows']
+        rows = rows.select { |row| row['key'] =~ regex } if regex        
+        rows.each do |row|
+            value = Field::Value.new(xtractr, row)
+            yield value
+        end
+    end
+    
+    # Fetch the values of the specified field for this packet. Even if there's 
+    # only a single value for the field, it's returned as an array of 1 element
+    #  packet.field('ip.ttl').each { |ttl| ... }
+    def [] name
+        result = xtractr.json "/api/packet/#{id}/field/#{name}"
+        return result['rows']
+    end
+    
+    # Extract just this packet and save it to the specified file as a pcap.
+    # You can also save a collection of packets using Packets#save or a
+    # collection of flows using Flows#save.
+    #   packet.save("foo.pcap")
+    def save filename
+        open(filename, "w") do |ios|
+            pcap = xtractr.get "api/packet/#{id}/pcap"
+            ios.write pcap
+        end
+        return self
+    end
+    
+    def inspect # :nodoc:
+        "#<pkt:#{id} #{src.address} > #{dst.address} #{service.name} #{title}"
+    end
+    
+    alias_method :each, :each_field
+    alias_method :fields, :entries
+    alias_method :field, :[]
+end
+end # Xtractr
+end # Mu