dap 0.0.1

data/lib/dap/filter/geoip.rb

@@ -0,0 +1,72 @@
+require 'geoip'
+
+module Dap
+module Filter
+
+module GeoIPLibrary
+  GEOIP_DIRS = [ 
+    File.expand_path( File.join( File.dirname(__FILE__), "..", "..", "..", "data")),
+    "/var/lib/geoip"
+  ]
+  GEOIP_CITY = %W{ geoip.dat geoip_city.dat GeoCity.dat IP_V4_CITY.dat GeoCityLite.dat }
+  GEOIP_ORGS = %W{ geoip_org.dat IP_V4_ORG.dat  }
+
+  @@geo_city = nil
+  @@geo_orgs = nil
+
+  GEOIP_DIRS.each do |d|
+    GEOIP_CITY.each do |f|
+      path = File.join(d, f)
+      if ::File.exist?(path)
+        @@geo_city = GeoIP::City.new(path)
+        break
+      end
+    end
+    GEOIP_ORGS.each do |f|
+      path = File.join(d, f)
+      if ::File.exist?( path )
+        @@geo_orgs = GeoIP::Organization.new(path)
+        break
+      end
+    end  
+  end  
+end
+
+
+#
+# Add GeoIP tags using the MaxMind GeoIP::City 
+#
+class FilterGeoIP
+  include BaseDecoder
+  include GeoIPLibrary
+  def decode(ip)
+    return unless @@geo_city
+    geo_hash = @@geo_city.look_up(ip)
+    return unless geo_hash
+    ret = {}
+    geo_hash.each_pair do |k,v|
+      next unless k
+      ret[k.to_s] = v.to_s
+    end
+    
+    ret
+  end
+end
+
+#
+# Add GeoIP tags using the MaxMind GeoIP::Organization database
+#
+class FilterGeoIPOrg
+  include BaseDecoder
+  include GeoIPLibrary
+  def decode(ip)
+    return unless @@geo_orgs
+    geo_hash = @@geo_orgs.look_up(ip)
+    return unless (geo_hash and geo_hash[:name])
+    { :org => geo_hash[:name] }
+  end
+end
+
+
+end
+end

data/lib/dap/filter/http.rb

@@ -0,0 +1,173 @@
+module Dap
+module Filter
+
+require 'htmlentities'
+require 'nokogiri'
+require 'uri'
+
+class FilterHTMLIframes
+  include Base
+
+  def process(doc)
+    out = []
+    self.opts.each_pair do |k,v|
+      next unless doc.has_key?(k)
+      extract(doc[k]).each do |url|
+        out << doc.merge({ 'iframe' => url })
+      end
+    end
+   out
+  end
+
+  def extract(data)
+    @coder ||= HTMLEntities.new
+    urls = []
+
+    data = data.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+    html = nil
+    begin
+      html = Nokogiri::HTML(data) do |conf|
+        conf.strict.noent
+      end
+    rescue ::Exception
+      return urls
+    end
+
+    html.xpath('//iframe').each do |e|
+      url = e['src']
+      next unless url
+      urls << url
+    end
+
+    urls
+  end
+end
+
+
+class FilterHTMLLinks
+  include Base
+
+  def process(doc)
+    out = []
+    self.opts.each_pair do |k,v|
+      next unless doc.has_key?(k)
+      extract(doc[k]).each do |link_info|
+        out << doc.merge(link_info)
+      end
+    end
+   out
+  end
+
+  def extract(data)
+    urls = []
+
+    data = data.encode('UTF-8', invalid: :replace, undef: :replace, replace: '')
+    html = nil
+    begin
+      html = Nokogiri::HTML(data) do |conf|
+        conf.strict.noent
+      end
+    rescue ::Exception
+     return urls
+    end
+
+    html.xpath('//*').each do |e|
+      url = e['href'] || e['src']
+      next unless url
+      urls << { 'link' => url, 'element' => e.name }
+    end
+
+    urls
+  end
+end
+
+class FilterDecodeURI
+  include BaseDecoder
+  def decode(data)
+    save = {}
+    uri  = URI.parse(data) rescue nil
+    return unless uri
+
+    save["host"] = uri.host if uri.host
+    save["port"] = uri.port.to_s if uri.port
+    save["path"] = uri.path if uri.path
+    save["query"]  = uri.query if uri.query
+    save["scheme"] = uri.scheme if uri.scheme
+    save["user"] = uri.user if uri.user
+    save["password"] = uri.password if uri.password
+
+    save
+  end
+end
+
+
+class FilterDecodeHTTPReply
+  include BaseDecoder
+
+  # TODO: Decode transfer-chunked responses
+  def decode(data)
+    lines = data.split(/\r?\n/)
+    resp  = lines.shift
+    save  = {}
+    return save if resp !~ /^HTTP\/\d+\.\d+\s+(\d+)\s+(.*)/
+
+    save["http_code"] = $1.to_i
+    save["http_message"] = $2.strip
+
+    clen = nil
+
+    while lines.length > 0
+      hline = lines.shift
+      case hline
+      when /^ETag:\s*(.*)/i
+        save["http_etag"] = $1
+
+      when /^Set-Cookie:\s*(.*)/i
+        bits = $1.gsub(/\;?\s*path=.*/i, '').gsub(/\;?\s*expires=.*/i, '').gsub(/\;\s*HttpOnly.*/, '')
+        save["http_cookie"] = bits.strip
+
+      when /^Server:\s*(.*)/i
+        save["http_server"] = $1.strip
+
+      when /^X-Powered-By:\s*(.*)/i
+        save["http_powered"] = $1.strip
+
+      when /^Date:\s*(.*)/i
+        d = DateTime.parse($1.strip) rescue nil
+        save["http_date"] = d.to_time.strftime("%Y%m%dT%H:%M:%S") if d
+          
+      when /^Last-modified:\s*(.*)/i
+        d = DateTime.parse($1.strip) rescue nil
+        save["http_modified"] = d.to_time.strftime("%Y%m%dT%H:%M:%S") if d
+
+      when /^Location:\s*(.*)/i
+        save["http_location"] = $1.strip  
+      
+      when /^WWW-Authenticate:\s*(.*)/i
+        save["http_auth"] = $1.strip
+
+      when /^Content-Length:\s*(.*)/i
+        clen = $1.strip.to_i
+
+      when ""
+        break
+      end
+    end
+
+    head, body = data.split(/\r?\n\r?\n/, 2)
+    
+    # Some buggy systems exclude the header entirely
+    body ||= head
+
+    save["http_body"] = body
+
+    if body =~ /<title>([^>]+)</min
+      save["http_title"] = $1.strip
+    end
+
+    save
+  end
+end
+
+end
+end

data/lib/dap/filter/names.rb

@@ -0,0 +1,151 @@
+module Dap
+module Filter
+
+MATCH_FQDN = /^([a-z0-9\_\-]+\.)+[a-z0-9\-]+\.?$/
+
+
+class FilterExtractHostname
+  include BaseDecoder
+  def decode(data)
+    data = data.strip.gsub(/.*\@/, '').gsub(/^\*+/, '').gsub(/^\.+/, '').gsub(/\.+$/, '').downcase
+    return unless data =~ MATCH_FQDN
+
+    # https://data.iana.org/TLD/tlds-alpha-by-domain.txt
+    return unless %w{ 
+      AC AD AE AERO AF AG AI AL AM AN AO AQ AR ARPA AS ASIA AT AU AW AX AZ BA BB BD BE BF BG BH BI BIKE BIZ
+      BJ BM BN BO BR BS BT BV BW BY BZ CA CAMERA CAT CC CD CF CG CH CI CK CL CLOTHING CM CN CO COM CONSTRUCTION
+      CONTRACTORS COOP CR CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EDU EE EG EQUIPMENT ER ES ESTATE ET EU FI FJ FK
+      FM FO FR GA GALLERY GB GD GE GF GG GH GI GL GM GN GOV GP GQ GR GRAPHICS GS GT GU GURU GW GY HK HM HN 
+      HOLDINGS HR HT HU ID IE IL IM IN INFO INT IO IQ IR IS IT JE JM JO JOBS JP KE KG KH KI KM KN KP KR KW KY KZ
+      LA LAND LB LC LI LIGHTING LK LR LS LT LU LV LY MA MC MD ME MG MH MIL MK ML MM MN MO MOBI MP MQ MR MS MT MU
+      MUSEUM MV MW MX MY MZ NA NAME NC NE NET NF NG NI NL NO NP NR NU NZ OM ORG PA PE PF PG PH PK PL PLUMBING PM
+      PN POST PR PRO PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SEXY SG SH SI SINGLES SJ SK SL SM SN SO SR ST
+      SU SV SX SY SZ TATTOO TC TD TECHNOLOGY TEL TF TG TH TJ TK TL TM TN TO TP TR TRAVEL TT TV TW TZ UA UG UK US
+      UY UZ VA VC VE VENTURES VG VI VN VOYAGE VU WF WS XN--3E0B707E XN--45BRJ9C XN--80AO21A XN--80ASEHDB 
+      XN--80ASWG XN--90A3AC XN--CLCHC0EA0B2G2A9GCD XN--FIQS8S XN--FIQZ9S XN--FPCRJ9C3D XN--FZC2C9E2C XN--GECRJ9C
+      XN--H2BRJ9C XN--J1AMH XN--J6W193G XN--KPRW13D XN--KPRY57D XN--L1ACC XN--LGBBAT1AD8J XN--MGB9AWBF 
+      XN--MGBA3A4F16A XN--MGBAAM7A8H XN--MGBAYH7GPA XN--MGBBH1A71E XN--MGBC0A9AZCG XN--MGBERP4A5D4AR 
+      XN--MGBX4CD0AB XN--NGBC5AZD XN--O3CW4H XN--OGBPF8FL XN--P1AI XN--PGBS0DH XN--S9BRJ9C XN--UNUP4Y 
+      XN--WGBH1C XN--WGBL6A XN--XKC2AL3HYE2A XN--XKC2DL3A5EE0H XN--YFRO4I67O XN--YGBI2AMMX XXX YE YT ZA ZM ZW
+      }.include?(data.split('.').last.upcase)
+
+    { 'hostname' => data }
+  end
+end
+
+class FilterSplitDomains
+  include Base
+  def process(doc)
+    lines = [ ]
+    self.opts.each_pair do |k,v|
+      if doc.has_key?(k)
+        expand(doc[k]).each do |line|
+          lines << doc.merge({ "#{k}.domain" => line })
+        end
+      end
+    end
+   lines.length == 0 ? [ doc ] : [ lines ]
+  end
+
+  def expand(data)
+    names = []
+    bits  = data.split('.')
+    while (bits.length > 1)
+      names << bits.join('.')
+      bits.shift
+    end
+    names
+  end
+end
+
+
+class FilterPrependSubdomains
+  include Base
+  def process(doc)
+    lines = [ ]
+    self.opts.each_pair do |k,v|
+      if doc.has_key?(k)
+        expand(doc[k], v).each do |line|
+          lines << doc.merge({ k => line })
+        end
+      end
+    end
+   lines.length == 0 ? [ ] : [ lines ]
+  end
+
+  def expand(data, names)
+    outp = [ data ]
+    bits = data.split(".")
+    subs = names.split(",")
+
+    # Avoid www.www.domain.tld and mail.www.domain.tld
+    return outp if subs.include?(bits.first)
+    subs.each do |sub|
+      outp << "#{sub}.#{data}"
+    end
+    
+    outp
+  end
+  
+end
+
+#
+# Acts like SplitDomains but strips out common dynamic IP RDNS formats
+#
+# XXX - Lots of work left to do
+#
+
+class FilterSplitNonDynamicDomains
+  include Base
+  def process(doc)
+    lines = [ ]
+    self.opts.each_pair do |k,v|
+      if doc.has_key?(k)
+        expand(doc[k]).each do |line|
+          lines << doc.merge({ "#{k}.domain" => line })
+        end
+      end
+    end
+   lines.length == 0 ? [ doc ] : [ lines ]
+  end
+
+  def expand(data)
+    names = []
+    data  = data.unpack("C*").pack("C*").
+      gsub(/.*ip\d+\.ip\d+\.ip\d+\.ip\d+\./, '').
+      gsub(/.*\d+[\_\-\.x]\d+[\_\-\.x]\d+[\_\-\.x]\d+[^\.]+/, '').
+      gsub(/.*node-[a-z0-9]+.*pool.*dynamic\./, '').
+      gsub(/.*[a-z][a-z]\d+\.[a-z]as[a-z0-9]+\./, '').
+      # cl223.001033200.technowave.ne.jp
+      gsub(/^cl\d+.[0-9]{6,14}\./, '').
+      # n157.s1117.m-zone.jp
+      gsub(/^n\d+.s\d+\.m-zone.jp/, 'm-zone.jp').
+      # u570054.xgsnu2.imtp.tachikawa.mopera.net
+      # s505207.xgsspn.imtp.tachikawa.spmode.ne.jp
+      gsub(/^[us]\d+.xgs[a-z0-9]+\.imtp/, 'imtp').
+      # tzbm6501209.tobizaru.jp
+      gsub(/^tzbm[0-9]{6,9}\./, '').
+      # ARennes-556-1-256-bdcst.w2-14.abo.wanadoo.fr
+      gsub(/.*\-\d+\-\d+\-\d+\-(net|bdcst)\./, '').
+      # bl19-128-119.dsl.telepac.pt
+      gsub(/.*\d+\-\d+\-\d+\.dsl/, 'dsl').
+      gsub(/.*pool\./, '').
+      gsub(/.*dynamic\./, '').
+      gsub(/.*static\./, '').
+      gsub(/.*dhcp[^\.]+\./, '').
+      gsub(/^\d{6,100}\./, '').
+      gsub(/^\.+/, '').
+      tr('^a-z0-9.-', '')
+
+    bits  = data.split('.')
+    while (bits.length > 1)
+      names << bits.join('.')
+      bits.shift
+    end
+    names
+  end
+end
+
+
+end
+end

data/lib/dap/filter/openssl.rb

@@ -0,0 +1,53 @@
+module Dap
+module Filter
+
+require 'openssl'
+
+class FilterDecodeX509
+  include BaseDecoder
+
+  def decode(data)
+    save = {}
+    cert = OpenSSL::X509::Certificate.new(data) rescue nil
+    return unless cert 
+
+    dnames = []
+    cert.subject.to_s.split("/").each do |bit|
+      var,val = bit.split("=", 2)
+      next unless (var and val)
+      var = var.to_s.downcase.strip
+      save["s_#{var}"] = val 
+      if var == "cn"
+        dnames << val
+      end
+    end
+
+    cert.issuer.to_s.split("/").each do |bit|
+      var,val = bit.split("=", 2)
+      next unless (var and val)
+      var = var.to_s.downcase.strip
+      save["i_#{var}"] = val 
+    end
+
+    cert.extensions.each do |e|
+      next unless e.to_s =~ /^([^\s]+)\s*=\s*(.*)/
+      var,val = $1,$2
+      var = var.to_s.downcase.strip
+      save["e_#{var}"] = val.strip
+
+      if var == "subjectaltname"
+        val.split(",").map{|x| x.gsub("DNS:", "").gsub("IP:", "").gsub("email:", "").strip }.each do |name|
+          dnames << name
+        end
+      end
+
+    end
+
+    save["names"] = dnames
+    save
+  end
+
+end
+
+end
+end