dap 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b61714c9b3553759bb7c726aad187acbf2adddf
-  data.tar.gz: e5f0ea147aa9a6f9fbcbb09854221b3d247eb467
+  metadata.gz: e8a027b00cf9225a71e0a599704a788c23200f2c
+  data.tar.gz: d5bf2864eb1ac5290a19bef4de11e76a3da5dcac
 SHA512:
-  metadata.gz: 53c0c68b19babf428a673063963122bb57f1e185a8205423f29c5e66ee9b4e2e3f2cfa5e905741a2bcf742358b2c5ddaef689edb862cb152a68ba453c7e8f592
-  data.tar.gz: fa2601b3d4bcaa99f0e3c4087e101e80f423a42228b95d53325c9bf31355aa8b186c3801f2a20b06a8cc1f4791ac6dfd1bc0b631fc829d4afa7e97b211fad25f
+  metadata.gz: 7e8aab164ce8b3be600320d2f9b32fe3da1ef76137efb6af3608bce035667b5b7422325a724dd232e105eabca7a47bdf81897a0ec2a6959bcffa45620f6c8154
+  data.tar.gz: 4ceecd71b00e4fef27dd33434ef9e55cb0f96ed38310fc93fafee9c1a2e60949525cb302bb0722fef69147732cf6b922ec391cecf2cb6b9331e83c9211a6d114

data/README.md

@@ -9,33 +9,62 @@ DAP reads data using an input plugin, transforms it through a series of filters,
 
 DAP was written to process terabyte-sized public scan datasets, such as those provided by https://scans.io/. Although DAP isn't particularly fast, it can be used across multiple cores (and machines) by splitting the input source and wrapping the execution with GNU Parallel.
 
-## Prerequisites
 
-DAP depends on GeoIP (http://dev.maxmind.com/geoip/legacy/downloadable/) to be able to append geographic metadata to analyzed datasets.  At least on Ubuntu, the libgeoip-dev package provides this capability.
+## Installation
 
-## Usage
+### Prerequisites
 
-See [Samples](https://github.com/rapid7/dap/tree/master/samples)
+DAP requires Ruby, and is best suited for systems with a relatively current version,
+preferably one installed and managed by
+[`rbenv`](https://github.com/rbenv/rbenv) or [`rvm`](https://rvm.io/).  Using
+system managed/installed Rubies is possible but fraught with peril.
 
-### Quick Setup for GeoIP Lookups
+DAP depends on [Maxmind's geoip database](http://dev.maxmind.com/geoip/legacy/downloadable/) to be able to append geographic metadata to analyzed datasets.  If you intend on using this capability, run the following as `root`:
 
+```bash
+mkdir -p /var/lib/geoip && cd /var/lib/geoip && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && gunzip GeoLiteCity.dat.gz && mv GeoLiteCity.dat geoip.dat
 ```
-$ git clone https://github.com/rapid7/dap.git
-$ cd dap
-$ gem install bundler
-$ bundle install
-$ sudo bash
-# mkdir -p /var/lib/geoip && cd /var/lib/geoip && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz && gunzip GeoLiteCity.dat.gz && mv GeoLiteCity.dat geoip.dat
+
+### Ubuntu
+
+```bash
+sudo apt-get install libgeoip-dev
+gem install dap
 ```
 
+### OS X
+
+```bash
+brew update
+brew install geoip
+gem install dap
+```
+
+## Usage
+
+In its simplest form, DAP takes input, applies zero or more filters which modify the input, and then outputs the result.  The input, filters and output are separated by plus signs (`+`).  As seen from `dap -h`:
+
+```
+Uage: dap  [input] + [filter] + [output]
+       --inputs
+       --outputs
+       --filters
+```
+
+To see which input/output formats are supported and what filters are available, run `dap --inputs`,`dap --outputs` or `dap --filters`, respectively.
+
+This example reads as input a single IP address from `STDIN` in line form, applies geo-ip transofmrations as a filter on that line, and then returns the output as JSON:
+
 ```
 $  echo 8.8.8.8 | bin/dap + lines + geo_ip line + json
 {"line":"8.8.8.8","line.country_code":"US","line.country_code3":"USA","line.country_name":"United States","line.latitude":"38.0","line.longitude":"-97.0"}
 ```
 
-Where dap gets fun is doing transforms, like just grabbing the country code:
+This example does the same, but only outputs the geo-ip country code:
+
 ```
 $  echo 8.8.8.8 | bin/dap + lines + geo_ip line + select line.country_code3 + lines
 USA
 ```
 
+There are also several examples of how to use DAP along with sample datasets [here](samples).

data/bin/dap

@@ -113,6 +113,7 @@ while true
   data = inp.read_record
   break if data == Dap::Input::Error::EOF
   next if data == Dap::Input::Error::Empty
+  next if data == Dap::Input::Error::InvalidFormat
 
   docs = [ data ]
   

data/lib/dap/filter/http.rb

@@ -4,6 +4,8 @@ module Filter
 require 'htmlentities'
 require 'shellwords'
 require 'uri'
+require 'zlib'
+require 'stringio'
 
 # Dirty element extractor, works around memory issues with Nokogiri
 module HTMLGhetto
@@ -191,9 +193,17 @@ class FilterDecodeHTTPReply
     # Some buggy systems exclude the header entirely
     body ||= head
 
+    if save["http_raw_headers"]["content-encoding"] == "gzip"
+      begin
+        gunzip = Zlib::GzipReader.new(StringIO.new(body))
+        body = gunzip.read.encode('UTF-8', :invalid=>:replace, :replace=>'?')
+        gunzip.close()
+      rescue
+      end
+    end
     save["http_body"] = body
 
-    if body =~ /<title>([^>]+)</min
+    if body =~ /<title>([^>]+)</mi
       save["http_title"] = $1.strip
     end
 

data/lib/dap/input.rb

@@ -2,22 +2,23 @@ module Dap
 module Input
 
   require 'oj'
-  
+
   #
   # Error codes for failed reads
-  # 
+  #
   module Error
     EOF   = :eof
     Empty = :empty
+    InvalidFormat = :invalid
   end
 
   module FileSource
-    
+
     attr_accessor :fd
 
     def open(file_name)
       close
-      self.fd = ['-', 'stdin', nil].include?(file_name) ? 
+      self.fd = ['-', 'stdin', nil].include?(file_name) ?
         $stdin : ::File.open(file_name, "rb")
     end
 
@@ -31,7 +32,7 @@ module Input
   # Line Input
   #
   class InputLines
-    
+
     include FileSource
 
     def initialize(args)
@@ -50,17 +51,23 @@ module Input
   # JSON Input (line-delimited records)
   #
   class InputJSON
-    
+
     include FileSource
 
     def initialize(args)
       self.open(args.first)
     end
 
-    def read_record  
+    def read_record
       line = self.fd.readline rescue nil
       return Error::EOF unless line
-      json = Oj.load(line.strip) rescue nil
+      begin
+        json = Oj.load(line.strip)
+      rescue
+        $stderr.puts "\nRecord is not valid JSON and will be skipped."
+        $stderr.puts line
+        return Error::InvalidFormat
+      end
       return Error::Empty unless json
       json
     end

data/lib/dap/proto/ldap.rb

@@ -166,6 +166,12 @@ class LDAP
       return [result_type, results]
     end
 
+    unless data.value && data.value.length > 1
+      result_type = 'Error'
+      results['errorMessage'] = 'parse_message: Invalid LDAP response (Empty Sequence)'
+      return [result_type, results]
+    end
+
     if data.value[1].tag == 4
       # SearchResultEntry found..
       result_type = 'SearchResultEntry'

data/lib/dap/version.rb

@@ -1,3 +1,3 @@
 module Dap
-  VERSION = "0.0.10"
+  VERSION = "0.0.11"
 end

data/spec/dap/filter/http_filter_spec.rb

@@ -0,0 +1,53 @@
+require 'zlib'
+
+describe Dap::Filter::FilterDecodeHTTPReply do
+  describe '.decode' do
+
+    let(:filter) { described_class.new(['data']) }
+
+
+    context 'decoding non-HTTP response' do
+      let(:decode) { filter.decode("This\r\nis\r\nnot\r\nHTTP\r\n\r\n") }
+      it 'returns an empty hash' do
+        expect(decode).to eq({})
+      end
+    end
+
+    context 'decoding uncompressed response' do
+      let(:decode) { filter.decode("HTTP/1.0 200 OK\r\nHeader1: value1\r\n\r\nstuff") }
+
+      it 'correctly sets status code' do
+        expect(decode['http_code']).to eq(200)
+      end
+
+      it 'correctly sets status message' do
+        expect(decode['http_message']).to eq('OK')
+      end
+
+      it 'correctly sets body' do
+        expect(decode['http_body']).to eq('stuff')
+      end
+
+      it 'correct extracts header(s)' do
+        expect(decode['http_raw_headers']).to eq({'header1' => 'value1'})
+      end
+    end
+
+    context 'decoding gzip compressed response' do
+      let(:body) {
+        io = StringIO.new
+        io.set_encoding('ASCII-8BIT')
+        gz = Zlib::GzipWriter.new(io)
+        gz.write('stuff')
+        gz.close
+        io.string
+      }
+      let(:decode) { filter.decode("HTTP/1.0 200 OK\r\nContent-encoding: gzip\r\n\r\n#{body}") }
+
+      it 'correctly decompresses body' do
+        expect(decode['http_body']).to eq('stuff')
+      end
+    end
+
+  end
+end

data/spec/dap/proto/ldap_proto_spec.rb

@@ -57,6 +57,8 @@ describe Dap::Proto::LDAP do
 
     data = original.pack('H*')
 
+    excessive_len = ['308480010000000000000000'].pack('H*')
+
     entry = ['3030020107642b040030273025040b6f626a656374436c6173'\
              '7331160403746f70040f4f70656e4c444150726f6f74445345']
 
@@ -90,6 +92,31 @@ describe Dap::Proto::LDAP do
         expect(split_messages.class).to eq(::Array)
       end
     end
+
+    context 'testing message length greater than total data length' do
+      let(:split_messages) { subject.split_messages(excessive_len) }
+      it 'returns Array as expected' do
+        expect(split_messages.class).to eq(::Array)
+      end
+
+      it 'returns empty Array as expected' do
+        expect(split_messages).to eq([])
+      end
+    end
+
+    context 'testing empty ASN.1 Sequence' do
+      hex = ['308400000000']
+      empty_seq = hex.pack('H*')
+
+      let(:split_messages) { subject.split_messages(empty_seq) }
+      it 'returns Array as expected' do
+        expect(split_messages.class).to eq(::Array)
+      end
+
+      it 'returns empty Array as expected' do
+        expect(split_messages).to eq([])
+      end
+    end
   end
 
   describe '.parse_ldapresult' do
@@ -209,6 +236,24 @@ describe Dap::Proto::LDAP do
       end
     end
 
+    context 'testing empty ASN.1 Sequence' do
+
+      data = OpenSSL::ASN1::Sequence.new([])
+
+      let(:parse_message) { subject.parse_message(data) }
+      it 'returns Array as expected' do
+        expect(parse_message.class).to eq(::Array)
+      end
+
+      it 'returns error value as expected' do
+        test_val = ['Error', {
+                      'errorMessage' =>
+                        'parse_message: Invalid LDAP response (Empty Sequence)'
+        }]
+        expect(parse_message).to eq(test_val)
+      end
+    end
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-02 00:00:00.000000000 Z
+date: 2016-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -219,6 +219,7 @@ files:
 - samples/ssl_certs_org.sh
 - samples/udp-netbios.csv.bz2
 - samples/udp-netbios.sh
+- spec/dap/filter/http_filter_spec.rb
 - spec/dap/filter/ldap_filter_spec.rb
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb
@@ -253,6 +254,7 @@ signing_key:
 specification_version: 4
 summary: 'DAP: The Data Analysis Pipeline'
 test_files:
+- spec/dap/filter/http_filter_spec.rb
 - spec/dap/filter/ldap_filter_spec.rb
 - spec/dap/proto/ipmi_spec.rb
 - spec/dap/proto/ldap_proto_spec.rb