dap 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7353b035e42f0ddd251e0e568ecd58cf22294869
-  data.tar.gz: fa161592d559ecb61c0ca8e03bb88a93377406f0
+  metadata.gz: a5979d47a46ba9175ad7041e9ae9e27281c0bbae
+  data.tar.gz: ce96da3ec587d1174890396b7e8cc0ec0d9e8789
 SHA512:
-  metadata.gz: 4f96b6c1e7032263ed23214ea4a5cd56a7ae16690f92e59321635cd3d05c74ae4d7f2735612e925e2517e471e7ec19187b64d8a83ecf2eb25bcfbc10f83c4f5b
-  data.tar.gz: b7f9c5bb9a947f7bec0c38f57bed0059be0b893bbdcc8087f8821db72aa1869069cf06eb999e72136b768fbadcff3ab3733237c8cea5979bdebb3bdf41d717bf
+  metadata.gz: f5ae71d2855a533c4d06060c416686e691fb5e93439a456b1b68c6c3b92443ba5790c3b5baf4e2dc2c04f51c69623dcfebc59a87bec13955fb319fc4bd5f736f
+  data.tar.gz: 39a3e52480b4970ca5f95df73090172243a857dc2902a251e913fbf3fc03250f6483baf52f5b8fb3e53701020adcc8531fbf73b0a8a214c5ad6abfbf72b4dad2

data/Gemfile

@@ -1,12 +1,11 @@
 source 'https://rubygems.org'
 
-gem 'nokogiri'
 gem 'oj'
 gem 'htmlentities'
 gem 'net-dns'
 gem 'bit-struct'
 gem 'geoip-c'
-gem 'recog', '>=1.0.15'
+gem 'recog', '>=2.0'
 
 group :test do
   gem 'rspec', '~> 3.1.0'

data/Gemfile.lock

@@ -28,7 +28,7 @@ GEM
     nokogiri (1.6.3.1)
       mini_portile (= 0.6.0)
     oj (2.10.2)
-    recog (1.0.15)
+    recog (2.0.2)
       nokogiri
     rspec (3.1.0)
       rspec-core (~> 3.1.0)
@@ -53,7 +53,6 @@ DEPENDENCIES
   geoip-c
   htmlentities
   net-dns
-  nokogiri
   oj
-  recog (>= 1.0.15)
+  recog (>= 2.0)
   rspec (~> 3.1.0)

data/data/vulndb.rb

@@ -80,4 +80,37 @@ SEARCHES = {
             ['7.0', 'sp4'] => ['CVE-2003-0230', 'CVE-2003-0231', 'CVE-2003-0232', 'CVE-2004-1560', 'CVE-2008-0085', 'CVE-2008-0086', 'CVE-2008-0106', 'CVE-2008-0107'],
         }
     }],
+
+    :http => [
+    #### ELASTICSEARCH RCE
+    {
+        # direct shellcommand elastic rce
+        :match => [
+            ['http.path', '/_search'],
+            ['http.body', 'script_fields'],
+            ['http.body', 'java.lang.Runtime'],
+            ['http.body', 'getRuntime()'],
+        ],
+        :cve => ['VULN-ELASTICSEARCH-RCE', 'CVE-2014-3120']
+    },{
+        # this just adds another tag as it's most likely done with metasploit
+        :match => [
+            ['http.path', '/_search'],
+            ['http.body', 'script_fields'],
+            ['http.body', 'metasploit.Payload'],
+        ],
+        :cve => ['VULN-ELASTICSEARCH-RCE', 'METASPLOIT']
+    }] + [
+    #### PHP CGI
+    {
+        :match => [
+            ['http.path', '/cgi-bin/php'],
+        ],
+        :cve => ['VULN-PHPCGI']
+    },{
+        :match => [
+            ['http.path', '/cgi-bin/authLogin.cgi'],
+        ],
+        :cve => ['VULN-QNAP-SHELLSHOCK']
+    }],
 }

data/lib/dap/filter/http.rb

@@ -2,11 +2,62 @@ module Dap
 module Filter
 
 require 'htmlentities'
-require 'nokogiri'
+require 'shellwords'
 require 'uri'
 
+# Dirty element extractor, works around memory issues with Nokogiri
+module HTMLGhetto
+  def extract_elements(data)
+    @coder ||= HTMLEntities.new
+    res = []
+    data.
+      to_s.
+      encode('UTF-8', invalid: :replace, undef: :replace, replace: '').
+      scan(/<([^>]+)>/m).each do |e|
+
+      e = e.first
+
+      # Skip closing tags
+      next if e[0,1] == "/"
+
+      # Get the name vs attributes
+      name, astr = e.split(/\s+/, 2).map{|x| x.to_s }
+      astr ||= ''
+
+      # Skip non-alpha elements
+      next unless name =~ /^[a-zA-Z]/
+
+      # Convert newlines to spaces & strip trailing />
+      astr = astr.gsub(/\n/, ' ').sub(/\/$/, '')
+
+      o = { name: name }
+
+      begin
+       Shellwords.shellwords(astr).each do |attr_str|
+          aname, avalue = attr_str.split('=', 2).map{|x| x.to_s.strip }
+          avalue = avalue.to_s.gsub(/^\"|"$/, '')
+          o[aname] = @coder.decode(avalue)
+        end
+      rescue ::Interrupt
+        raise $!
+      rescue ::Exception
+        # If shellwords couldn't parse it, split on space instead
+        astr.to_s.split(/\s+/).each do |attr_str|
+          aname, avalue = attr_str.split('=', 2).map{|x| x.to_s.strip }
+          avalue = avalue.to_s.gsub(/^\"|"$/, '')
+          o[aname] = @coder.decode(avalue)
+        end
+      end
+      res << o
+    end
+
+    res
+  end
+end
+
 class FilterHTMLIframes
   include Base
+  include HTMLGhetto
 
   def process(doc)
     out = []
@@ -20,25 +71,11 @@ class FilterHTMLIframes
   end
 
   def extract(data)
-    @coder ||= HTMLEntities.new
-    urls = []
-
-    data = data.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
-    html = nil
-    begin
-      html = Nokogiri::HTML(data) do |conf|
-        conf.strict.noent
-      end
-    rescue ::Exception
-      return urls
-    end
-
-    html.xpath('//iframe').each do |e|
+    extract_elements(data).select{|x| x[:name] == 'iframe'}.each do |e|
       url = e['src']
-      next unless url
+      next unless (url && url.length > 0)
       urls << url
     end
-
     urls
   end
 end
@@ -46,6 +83,7 @@ end
 
 class FilterHTMLLinks
   include Base
+  include HTMLGhetto
 
   def process(doc)
     out = []
@@ -61,20 +99,10 @@ class FilterHTMLLinks
   def extract(data)
     urls = []
 
-    data = data.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
-    html = nil
-    begin
-      html = Nokogiri::HTML(data) do |conf|
-        conf.strict.noent
-      end
-    rescue ::Exception
-     return urls
-    end
-
-    html.xpath('//*').each do |e|
+    extract_elements(data).each do |e|
       url = e['href'] || e['src']
-      next unless url
-      urls << { 'link' => url, 'element' => e.name }
+      next unless (url && url.length > 0)
+      urls << { 'link' => url, 'element' => e[:name] }
     end
 
     urls
@@ -136,14 +164,14 @@ class FilterDecodeHTTPReply
       when /^Date:\s*(.*)/i
         d = DateTime.parse($1.strip) rescue nil
         save["http_date"] = d.to_time.strftime("%Y%m%dT%H:%M:%S") if d
-          
+
       when /^Last-modified:\s*(.*)/i
         d = DateTime.parse($1.strip) rescue nil
         save["http_modified"] = d.to_time.strftime("%Y%m%dT%H:%M:%S") if d
 
       when /^Location:\s*(.*)/i
-        save["http_location"] = $1.strip  
-      
+        save["http_location"] = $1.strip
+
       when /^WWW-Authenticate:\s*(.*)/i
         save["http_auth"] = $1.strip
 
@@ -159,7 +187,7 @@ class FilterDecodeHTTPReply
     end
 
     head, body = data.split(/\r?\n\r?\n/, 2)
-    
+
     # Some buggy systems exclude the header entirely
     body ||= head
 

data/lib/dap/filter/vulnmatch.rb

@@ -64,5 +64,145 @@ class FilterVulnMatchMSSQL
   end
 end
 
+class FilterVulnMatchHTTP
+  include Base
+  include BaseVulnMatch
+
+  def check_shellshock(doc)
+    if not doc["http.headers"]
+      return []
+    end
+
+    h = doc["http.headers"]
+    sspattern = /\(\)\s*{\s*:;\s*};/
+
+    if h["user-agent"] and h["user-agent"] =~ sspattern
+      return ['VULN-SHELLSHOCK', 'CVE-2014-6271']
+    end
+
+    if h["referrer"] and h["referrer"] =~ sspattern
+      return ['VULN-SHELLSHOCK', 'CVE-2014-6271']
+    end
+
+    return []
+  end
+
+  def check_elastic(doc)
+    if not doc['http.path']
+      return []
+    end
+    if not doc['http.path'] == '/_search'
+      return []
+    end
+
+    input = doc['http.url']
+    if doc['http.method'] == "POST"
+      input = doc['http.body']
+    end
+
+    if not input.match("script_fields")
+      return []
+    end
+
+    out = ['VULN-ELASTICSEARCH-RCE', 'CVE-2014-3120']
+    if input.match("Runtime") and input.match("getRuntime()")
+      out += ["EXEC-SHELLCMD"]
+    end
+
+    if input.match("FileOutputStream") and input.match("URLClassLoader")
+      out += ["EXEC-JAVA-CLASS"]
+    end
+
+    if input.match("getDeclaredConstructor")
+      out += ['CVE-2015-1427']
+    end
+
+    if input.match("metasploit.Payload")
+      out += ['METASPLOIT']
+    end
+
+    return out
+  end
+
+  def process(doc)
+    vulns = []
+    if doc['vulnerability']
+      vulns |= doc['vulnerability']
+    end
+
+    vulns |= check_elastic(doc)
+    vulns |= check_shellshock(doc)
+
+    # see vulndb.rb, allows for simple matches to be added quickly
+    SEARCHES[:http].each do | entry |
+      success = true
+
+      # all matches must go through
+      entry[:match].each do | k, v |
+        if not doc[k]
+          success = false
+        else
+          m = doc[k].match(v)
+          if not m
+            success = false
+          end
+        end
+
+        if not success
+          break
+        end
+      end
+
+      if success
+        vulns |= entry[:cve]
+      end
+    end
+
+    if vulns != []
+      doc['vulnerability'] = vulns
+    end
+
+    [ doc ]
+  end
+end
+
+class FilterGenericSetMatch
+  include Base
+  attr_accessor :matchset
+
+  def initialize(args)
+    self.opts = {}
+    args.each do |arg|
+        k,v = arg.split("=", 2)
+        self.opts[k] = v
+    end
+    self.name = Dap::Factory.name_from_class(self.class)
+
+    fail "Expected key and set arguments to #{self.name} but got #{self.opts}" unless self.opts.has_key?("key") and self.opts.has_key?("set")
+
+    self.matchset = {}
+    File.readlines(self.opts["set"]).each do |line|
+      self.matchset[line.chomp] = nil
+    end
+  end
+
+  def process(doc)
+    if doc.has_key?(self.opts["key"])
+      if doc[self.opts["key"]].kind_of?(Array)
+        doc[self.opts["key"]].each do |val|
+          if self.matchset.has_key?(val)
+            return [ doc ]
+          end
+        end
+      else
+        if self.matchset.has_key?(doc[self.opts["key"]])
+          return [ doc ]
+        end
+      end
+    end
+    [ ]
+  end
+end
+
 end
 end

data/lib/dap/version.rb

@@ -1,3 +1,3 @@
 module Dap
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dap
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-10 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -241,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: 'DAP: The Data Analysis Pipeline'