dorothy2 0.0.3 → 1.0.0

data/lib/mu/xtractr/test/tc_views.rb

@@ -0,0 +1,118 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'mu/xtractr'
+require 'test/unit'
+
+module Mu
+class Xtractr
+class Views
+    class Count
+    class Test < Test::Unit::TestCase
+        attr_reader :xtractr
+        attr_reader :count
+    
+        def setup
+            @xtractr = Xtractr.new
+            @count = xtractr.flows('flow.service:DNS').count('dns.qry.name').first
+        end
+    
+        def test_attributes
+            assert_kind_of(Field, count.field)
+            assert_equal('ax.search.itunes.apple.com', count.value)
+            assert_equal(8, count.count)
+        end
+        
+        def test_object
+            object = count.object
+            assert_kind_of(Field::Value, object)
+            assert_equal('dns.qry.name', object.field.name)
+            assert_equal('ax.search.itunes.apple.com', object.value)
+        end
+    
+        def test_packets
+            packets = count.packets
+            assert_equal("dns.qry.name:\"ax.search.itunes.apple.com\"", packets.q)
+        end
+    
+        def test_each_packet
+            count.each_packet do |pkt| 
+                assert_kind_of(Packet, pkt)
+                values = pkt['dns.qry.name']
+                assert_equal(1, values.size)
+                assert_equal('ax.search.itunes.apple.com', values[0])
+            end
+        end
+        
+        def test_sum
+            sums = count.object.sum('pkt.src', 'pkt.length')
+            assert_equal(2, sums.length)
+        end
+    
+        def test_inspect
+            assert_nothing_raised { count.inspect }
+        end
+    end
+    end # Count
+    
+    class Sum
+    class Test < Test::Unit::TestCase
+        attr_reader :xtractr
+        attr_reader :sum
+    
+        def setup
+            @xtractr = Xtractr.new
+            @sum = xtractr.flows('flow.service:DNS').sum('dns.qry.name', 'flow.bytes').first
+        end
+    
+        def test_attributes
+            assert_kind_of(Field, sum.field)
+            assert_equal('ax.search.itunes.apple.com', sum.value)
+            assert_equal(1220, sum.sum)
+        end
+        
+        def test_object
+            object = sum.object
+            assert_kind_of(Field::Value, object)
+            assert_equal('dns.qry.name', object.field.name)
+            assert_equal('ax.search.itunes.apple.com', object.value)
+        end
+    
+        def test_packets
+            packets = sum.packets
+            assert_equal("dns.qry.name:\"ax.search.itunes.apple.com\"", packets.q)
+        end
+    
+        def test_each_packet
+            sum.each_packet do |pkt| 
+                assert_kind_of(Packet, pkt)
+                values = pkt['dns.qry.name']
+                assert_equal(1, values.size)
+                assert_equal('ax.search.itunes.apple.com', values[0])
+            end
+        end
+        
+        def test_count
+            counts = sum.object.count('pkt.service')
+            assert_equal(1, counts.length)
+        end
+    
+        def test_inspect
+            assert_nothing_raised { sum.inspect }
+        end
+    end
+    end # Sum
+end # Views
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/test/tc_xtractr.rb

@@ -0,0 +1,151 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'mu/xtractr'
+require 'test/unit'
+
+module Mu
+class Xtractr
+class Test < Test::Unit::TestCase
+    attr_reader :xtractr
+    
+    def setup
+        @xtractr = Xtractr.new
+    end
+    
+    def test_about
+        about = xtractr.about
+        assert_equal(1778, about.packets)
+        assert_equal(32, about.flows)
+        assert_equal(12, about.hosts)
+        assert_equal(5, about.services)
+        assert_equal(171.172, about.duration)
+    end
+    
+    def test_hosts
+        hosts = xtractr.hosts
+        assert_equal(12, hosts.size)
+        hosts.each { |host| assert_instance_of(Host, host) }
+        assert_equal(8, xtractr.hosts(/^8.18/).size)
+        assert_equal(8, xtractr.hosts('8.18').size)
+        assert_equal(1, xtractr.hosts(/^4.2/).size)
+        assert_equal(1, xtractr.hosts('4.2').size)
+    end
+    
+    def test_host
+        [ 
+            '4.2.2.1',
+            '8.18.65.67',
+            '8.18.65.32',
+            '8.18.65.58',
+            '8.18.65.82',
+            '8.18.65.27',
+            '8.18.65.10',
+            '8.18.65.88',
+            '8.18.65.89',
+            '66.235.132.121',
+            '192.168.1.10',
+            '224.0.0.251',
+        ].each do |address|
+            assert_nothing_raised { xtractr.host address }
+        end
+        
+        assert_raise(ArgumentError) { xtractr.host '1.1.1.1' }        
+    end
+    
+    def test_services
+        services = xtractr.services
+        assert_equal(5, services.size)
+        services.each { |service| assert_instance_of(Service, service) }
+        assert_equal(2, xtractr.services(/HTTP/).size)
+        assert_equal(2, xtractr.services('HTTP').size)
+        assert_equal(2, xtractr.services('http').size)
+    end
+    
+    def test_service
+        [ 'DNS', 'TCP', 'HTTP', 'HTTP/XML', 'MDNS' ].each do |name|
+            assert_nothing_raised { xtractr.service name }
+            assert_nothing_raised { xtractr.service name.downcase }
+        end
+        
+        assert_raise(ArgumentError) { xtractr.service 'blah' }
+    end
+    
+    def test_fields
+        fields = xtractr.fields
+        assert_equal(170, fields.size)
+        fields.each { |field| assert_instance_of(Field, field) }
+        assert_equal(12, xtractr.fields(/^pkt\./).size)
+        assert_equal(12, xtractr.fields("PKT.").size)
+        assert_equal(12, xtractr.fields("pkt.").size)
+    end
+    
+    def test_field
+        [ 
+            'pkt.src', 'pkt.dst', 'pkt.flow', 'pkt.id', 'pkt.pcap', 'pkt.first', 
+            'pkt.dir', 'pkt.time', 'pkt.offset', 'pkt.length', 'pkt.service',
+            'pkt.title'
+        ].each do |name|
+            assert_nothing_raised { xtractr.field name }
+        end
+        assert_raise(ArgumentError) { xtractr.field 'blah' }
+    end
+    
+    def test_flows
+        flows = xtractr.flows
+        assert_kind_of(Flows, flows)
+        assert_equal('*', flows.q)
+        
+        flows = xtractr.flows 'blah:foo'
+        assert_equal('blah:foo', flows.q)
+        
+        flows = xtractr.flows 1..10
+        assert_equal('flow.id:[1 10]', flows.q)
+        
+        flows = xtractr.flows 1...10
+        assert_equal('flow.id:[1 9]', flows.q)
+    end
+    
+    def test_flow
+        flow = xtractr.flow 1
+        assert_kind_of(Flow, flow)
+        
+        assert_raise(ArgumentError) do
+            flow = xtractr.flow xtractr.about.flows+1
+        end
+    end
+    
+    def test_packets
+        packets = xtractr.packets
+        assert_kind_of(Packets, packets)
+        assert_equal('*', packets.q)
+        
+        packets = xtractr.packets 'blah:foo'
+        assert_equal('blah:foo', packets.q)
+        
+        packets = xtractr.packets 1..10
+        assert_equal('pkt.id:[1 10]', packets.q)
+        
+        packets = xtractr.packets 1...10
+        assert_equal('pkt.id:[1 9]', packets.q)
+    end
+    
+    def test_packet
+        pkt = xtractr.packet 1
+        assert_kind_of(Packet, pkt)
+        assert_equal(1, pkt.id)
+    end
+end
+end # Xtractr
+end # Mu

data/lib/mu/xtractr/test/test.rb

@@ -0,0 +1,19 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+require 'test/unit'
+
+Dir.glob('mu/xtractr/test/**/tc*.rb').each do |file|
+    require file
+end

data/lib/mu/xtractr/views.rb

@@ -0,0 +1,204 @@
+# "THE BEER-WARE LICENSE" (Revision 42):
+# Mu[http://www.mudynamics.com] wrote this file. As long as you retain this 
+# notice you can do whatever you want with this stuff. If we meet some day, 
+# and you think this stuff is worth it, you can buy us a beer in return. 
+#
+# All about pcapr
+# * http://www.pcapr.net
+# * http://groups.google.com/group/pcapr-forum
+# * http://twitter.com/pcapr
+#
+# Mu Dynamics
+# * http://www.mudynamics.com
+# * http://labs.mudynamics.com
+
+module Mu
+class Xtractr
+# See http://labs.mudynamics.com/2009/04/03/interactive-couchdb/ for a quick
+# tutorial on how Map/Reduce works.
+class Views # :nodoc:
+    # = Count
+    # Count contains the results of doing a map/reduce on either flows or
+    # packets. Each count contains the field on which the map/reduce was
+    # performed, the unique value as all as the count of that value in the
+    # flows or packets. For example to count the unique source IP address of
+    # HTTP flows in the first five minutes of the index, you would do:
+    #
+    #  xtractr.flows('flow.service:HTTP flow.duration:[1 300]').count('flow.src')
+    class Count
+        attr_reader :xtractr # :nodoc:
+        
+        # Returns the field used for counting.
+        attr_reader :field
+        
+        # Returns the unique value of the field.
+        attr_reader :value
+        
+        # Returns the count of the field/value.
+        attr_reader :count
+        
+        def initialize xtractr, field, value, count # :nodoc:
+            @xtractr = xtractr
+            @field   = field
+            @value   = value
+            @count   = count
+        end
+        
+        # Returns a Field::Value object that can be used for further method
+        # chaining.
+        #  xtractr.flows.count('flow.src').first.object.count('flow.service')
+        def object
+            Field::Value.new xtractr, "key" => field.name, "value" => value
+        end
+        
+        # Fetch the list of packets that contain this field value.
+        #  xtractr.flows.count('flow.src').first.packets.each { |pkt ... }
+        def packets q=nil
+            object.packets q
+        end
+        
+        # Iterate over each packet that contains this field value.
+        #  xtractr.flows.count('flow.src').first.each_packet { |pkt ... }
+        def each_packet(q=nil, &blk) # :yields: packet
+            packets(q).each(&blk)
+            return self
+        end
+        
+        # Sum the numeric values of vfield, keyed by the unique values of
+        # kfield. This is used for method chaining.
+        #  xtractr.flows.count('flow.src').first.sum('flow.service', 'flow.bytes')
+        def sum kfield, vfield
+            object.sum kfield, vfield
+        end
+        
+        def inspect # :nodoc:
+            "#<count #{value} #{count}>"
+        end
+    end
+    
+    # = Sum
+    # Sum contains the results of doing a map/reduce on either flows or
+    # packets. Each sum contains the field on which the map/reduce was
+    # performed, the unique value as all as the sum of that value in the
+    # flows or packets. For example to count the bytes sent in HTTP flows
+    # keyed by the source IP address, you would do:
+    #
+    #  xtractr.flows('flow.service:HTTP').count('flow.src', 'flow.bytes')
+    class Sum
+        attr_reader :xtractr # :nodoc:
+        
+        # Returns the field used for summing.
+        attr_reader :field
+        
+        # Returns the unique value used as the map/reduce key.
+        attr_reader :value
+        
+        # Returns the aggregate computed sum.
+        attr_reader :sum
+        
+        def initialize xtractr, field, value, sum # :nodoc:
+            @xtractr = xtractr
+            @field   = field
+            @value   = value
+            @sum     = sum
+        end
+        
+        # Returns a Field::Value object that can be used for further method
+        # chaining. In the following example, we first compute the top talkers
+        # (based on the bytes sent) and then use the topmost talker to count
+        # the list of unique services.
+        #  xtractr.flows.sum('flow.src', 'flow.bytes').first.object.count('flow.service')
+        def object
+            Field::Value.new xtractr, "key" => field.name, "value" => value
+        end
+        
+        # Fetch the list of packets that contain this field value.
+        #  xtractr.flows.sum('flow.src', 'flow.bytes').first.packets.each { |pkt ... }
+        def packets q=nil
+            object.packets q
+        end
+        
+        # Iterate over each packet that contains this field value.
+        #  xtractr.flows.sum('flow.src', 'flow.bytes').first.each_packet { |pkt ... }
+        def each_packet q=nil, &blk
+            packets(q).each(&blk)
+            return self
+        end
+        
+        # Count the unique values of the specified field amongst all the packets
+        # that matched the query.
+        #  xtractr.flows.sum('flow.src', 'flow.bytes').first.count('flow.service')
+        def count _field
+            object.count _field
+        end
+        
+        def inspect # :nodoc:
+            "#<sum #{value} #{sum}>"
+        end
+    end
+    
+    def self.count xtractr, field, url, opts={} # :nodoc:
+        field = Field.new(xtractr, field) if field.is_a? String
+        name = field.name.gsub /^(pkt|flow)\./, ''
+        _opts = opts.dup
+        _opts[:r] = <<-EOS
+            ({
+                map: function(_pf) {
+                    _pf.values("#{name}", function(_value) {
+                        if (_value) {
+                            if (typeof(_value) === 'string') {
+                                if (_value.length > 1024) {
+                                    _value = _value.slice(0,1024);
+                                }
+                            }
+                            emit(_value, 1);
+                        }
+                    });
+                },
+                reduce: function(_key, _values) {
+                    return sum(_values);
+                }
+            })
+        EOS
+        result = xtractr.json url, _opts
+        result['rows'].map do |row| 
+            Views::Count.new(xtractr, field, row['key'], row['value'])
+        end.sort { |a, b| b.count <=> a.count }
+    end
+    
+    def self.sum xtractr, kfield, vfield, url, opts={} # :nodoc:
+        kfield = Field.new(xtractr, kfield) if kfield.is_a? String
+        vfield = Field.new(xtractr, vfield) if vfield.is_a? String
+        kname = kfield.name.gsub /^(pkt|flow)\./, ''
+        vname = vfield.name.gsub /^(pkt|flow)\./, ''
+        _opts = opts.dup
+        _opts[:r] = <<-EOS
+            ({
+                map: function(_pf) {
+                    var _key = _pf["#{kname}"];
+                    if (_key) {
+                        if (typeof(_key) === 'string') {
+                            if (_key.length > 1024) {
+                                _key = _key.slice(0,1024);
+                            }
+                        }
+                        _pf.values("#{vname}", function(_val) {
+                            if (typeof(_val) === 'number') {
+                                emit(_key, _val);
+                            }
+                        });
+                    }
+                },
+                reduce: function(_key, _values) {
+                    return sum(_values);
+                }
+            })
+        EOS
+        result = xtractr.json url, _opts
+        result['rows'].map do |row| 
+            Views::Sum.new(xtractr, kfield, row['key'], row['value'])
+        end.sort { |a, b| b.sum <=> a.sum }
+    end
+end
+end # Xtractr
+end # Mu