intrigue-ident 0.4 → 0.6

data/lib/checks/tableau.rb

@@ -0,0 +1,26 @@
+module Intrigue
+module Ident
+module Check
+    class Tableau < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "application",
+            :vendor => "Tableau",
+            :product => "Tableau",
+            :match_details => "Tableau Server",
+            :version => nil,
+            :references => ["https://community.tableau.com/thread/165653"],
+            :match_type => :content_body,
+            :match_content =>  /<meta name="vizportal-config" data-buildId=/i,
+            :examples => ["http://137.154.26.56:80"],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/lib/checks/team_city.rb

@@ -3,15 +3,17 @@ module Ident
 module Check
     class TeamCity < Intrigue::Ident::Check::Base
 
-      def generate_checks(uri)
+      def generate_checks(url)
         [
           {
-            :name => "TeamCity Continuous Integration",
-            :description => "TeamCity Continuous Integration",
+            :type => "application",
+            :vendor => "TeamCity",
+            :product => "TeamCity",
+            :match_details => "TeamCity Continuous Integration",
             :version => nil,
-            :type => :content_body,
-            :content => /icons\/teamcity.black.svg/i,
-            :paths => ["#{uri}"]
+            :match_type => :content_body,
+            :match_content =>  /icons\/teamcity.black.svg/i,
+            :paths => ["#{url}"]
           }
         ]
       end

data/lib/checks/telerik.rb

@@ -3,18 +3,33 @@ module Ident
 module Check
     class Telerik < Intrigue::Ident::Check::Base
 
-      def generate_checks(uri)
+      def generate_checks(url)
         [
           {
-            :name => "Telerik Sitefinity",
-            :description => "Telerik Sitefinity is an ASP.NET 2.0-based Content Management System (CMS)",
+            :type => "application",
+            :vendor => "Telerik",
+            :product => "Sitefinity",
+            :match_details => "Telerik Sitefinity is an ASP.NET 2.0-based Content Management System (CMS)",
             :url => "https://www.sitefinity.com/",
             :version => nil,
-            :type => :content_body,
-            :content => /Telerik.Sitefinity.Resources/,
-            :dynamic_version => lambda { |x| x.body.match(/Version=([\d\.]+),/).captures[0] },
-            :verify_sites => [],
-            :paths => ["#{uri}"]
+            :match_type => :content_body,
+            :match_content =>  /Telerik.Sitefinity.Resources/,
+            :dynamic_version => lambda { |x|  _first_body_capture x, /Version=([\d\.]+),/ },
+            :examples => [],
+            :paths => ["#{url}"]
+          },
+          {
+            :type => "application",
+            :vendor => "Telerik",
+            :product => "Sitefinity",
+            :match_details => "Detect Telerik via a meta generator tag",
+            :url => "https://www.sitefinity.com/",
+            :version => nil,
+            :match_type => :content_body,
+            :match_content =>  /<meta\ name=\"Generator\"\ content=\"Sitefinity/,
+            :dynamic_version => lambda { |x| _first_body_capture x, /<meta name=\"Generator\" content=\"Sitefinity (.*)\ \/><link/ },
+            :examples => [],
+            :paths => ["#{url}"]
           }
         ]
       end

data/lib/checks/varnish.rb

@@ -3,20 +3,17 @@ module Ident
 module Check
     class Varnish < Intrigue::Ident::Check::Base
 
-      def generate_checks(uri)
+      def generate_checks(url)
         [
           {
-            :name => "Varnish",
-            :description => "Varnish Proxy",
+            :type => "application",
+            :vendor =>"Varnish",
+            :product =>"Varnish",
+            :match_details =>"Varnish Proxy",
             :version => nil,
-            :type => :content_headers,
-            :content => /via: [0-9]\.[0-9] varnish/i,
-            :dynamic_version => lambda{ |x|
-              m = nil
-              x.each_header{|h,v| m = v if (h == "via" && v =~ /varnish/) }
-              m.gsub("varnish ","") if m
-            },
-            :paths => ["#{uri}"]
+            :match_type => :content_headers,
+            :match_content =>  /via: [0-9]\.[0-9] varnish/i,
+            :paths => ["#{url}"]
           }
         ]
       end

data/lib/checks/vmware.rb

@@ -0,0 +1,38 @@
+module Intrigue
+module Ident
+module Check
+    class Vmware < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "application",
+            :vendor => "VMWare",
+            :tags => ["tech:hypervisor"],
+            :product =>"ESXi",
+            :match_details =>"unique page string",
+            :version => nil,
+            :match_type => :content_body,
+            :match_content => /document.write\(\"<title>\"\ \+\ ID_EESX_Welcome/,
+            :paths => ["#{url}"],
+            :verify => ["aWJtI0ludHJpZ3VlOjpFbnRpdHk6OlVyaSNodHRwOi8vMTIuNDIuMjA1LjEyNzo4MA=="],
+            :examples => ["http://12.42.205.127:80"]
+          },
+          {
+            :type => "application",
+            :vendor => "VMWare",
+            :tags => ["tech:hypervisor"],
+            :product =>"Horizon",
+            :match_details =>"page title",
+            :version => nil,
+            :match_type => :content_body,
+            :match_content =>  /<title>VMware Horizon/,
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+end
+end
+end

data/lib/checks/webmin.rb

@@ -0,0 +1,41 @@
+module Intrigue
+module Ident
+module Check
+  class Webmin < Intrigue::Ident::Check::Base
+
+    def generate_checks(url)
+      [
+        {
+          :type => "application",
+          :vendor =>"Webmin",
+          :product =>"MiniServ",
+          :match_details => "server header",
+          :match_type => :content_headers,
+          :references => [],
+          :match_content => /server: MiniServ/,
+          :version => nil,
+          :dynamic_version => lambda {|x| _first_header_capture(x,/server: MiniServ\/(.*)/)},
+          :examples => ["http://158.85.208.126:8080"],
+          :verify => ["aWJtI0ludHJpZ3VlOjpFbnRpdHk6OlVyaSNodHRwOi8vMTU4Ljg1LjIwOC4xMjY6ODA4MA=="],
+          :paths => ["#{url}"]
+        },
+        {
+          :type => "application",
+          :vendor =>"Webmin",
+          :product =>"Webmin",
+          :match_details => "page title",
+          :match_type => :content_body,
+          :references => [],
+          :match_content => /<title>Login to Webmin/,
+          :version => nil,
+          :examples => ["http://158.85.208.126:8080"],
+          :verify => ["aWJtI0ludHJpZ3VlOjpFbnRpdHk6OlVyaSNodHRwOi8vMTU4Ljg1LjIwOC4xMjY6ODA4MA=="],
+          :paths => ["#{url}"]
+        }
+      ]
+    end
+
+  end
+end
+end
+end

data/lib/checks/wp_engine.rb

@@ -3,15 +3,19 @@ module Ident
 module Check
     class WpEngine < Intrigue::Ident::Check::Base
 
-      def generate_checks(uri)
+      def generate_checks(url)
         [
           {
-            :name => "WPEngine",
-            :description => "WPEngine - Access site by IP",
+            :type => "service",
+            :vendor =>"WPEngine",
+            :tags => ["hosting_provider"],
+            :product =>"WPEngine",
+            :match_details =>"WPEngine - Access site by IP",
             :version => nil,
-            :type => :content_body,
-            :content => /This domain is successfully pointed at WP Engine, but is not configured for an account on our platform./,
-            :paths => ["#{uri}"]
+            :match_type => :content_body,
+            :match_content =>  /This domain is successfully pointed at WP Engine, but is not configured for an account on our platform./,
+            :hide => true,
+            :paths => ["#{url}"]
           }
         ]
       end

data/lib/checks/yaws.rb

@@ -0,0 +1,29 @@
+module Intrigue
+module Ident
+module Check
+    class Yaws < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "application",
+            :vendor =>"Yaws",
+            :product =>"Yaws",
+            :match_details =>"server header",
+            :references => ["https://en.wikipedia.org/wiki/Yaws_(web_server)"],
+            :match_type => :content_headers,
+            :match_content =>  /server: Yaws/i,
+            :dynamic_version => lambda { |x|
+              _first_header_capture(x,/server: Yaws (.*)/i)
+            },
+            :examples => ["https://158.85.224.176:443"],
+            :verify => ["aWJtI0ludHJpZ3VlOjpFbnRpdHk6OlVyaSNodHRwczovLzE1OC44NS4yMjQuMTc2OjQ0Mw=="],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/lib/checks/zeit.rb

@@ -0,0 +1,28 @@
+module Intrigue
+module Ident
+module Check
+    class Zeit < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "application",
+            :vendor =>"Zeit",
+            :product =>"Next.js",
+            :match_details =>"x-powered-by header",
+            :references => ["https://zeit.co/blog/next"],
+            :match_type => :content_headers,
+            :match_content =>  /x-powered-by: Next.js/i,
+            :dynamic_version => lambda { |x|
+              _first_header_capture(x,/sx-powered-by: Next.js\ (.*)/i)
+            },
+            :examples => ["http://static.invisionapp.com:80"],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/lib/checks/zendesk.rb

@@ -0,0 +1,39 @@
+module Intrigue
+module Ident
+module Check
+    class Zendesk < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "service",
+            :vendor =>"Zendesk",
+            :product =>"Zendesk",
+            :match_details =>"unique header",
+            :references => [],
+            :match_type => :content_headers,
+            :match_content =>  /^x-zendesk-origin-server:.*$/i,
+            :examples => ["http://help.etsy.com:80"],
+            :verify => ["ZXRzeSNJbnRyaWd1ZTo6RW50aXR5OjpVcmkjaHR0cDovL2hlbHAuZXRzeS5jb206ODA="],
+            :paths => ["#{url}"]
+          },
+          { # TODO - this might catch valid (closed) helpdesk uris too.
+            :type => "service",
+            :vendor =>"Zendesk",
+            :product =>"Zendesk",
+            :match_details =>"zendesk access by IP / invalid hostname",
+            :references => [],
+            :hide => true,
+            :match_type => :content_body,
+            :match_content =>  /<title>Help Center Closed \| Zendesk/i,
+            :examples => ["http://192.161.147.1:80"],
+            :verify => ["a2VubmFzZWN1cml0eSNJbnRyaWd1ZTo6RW50aXR5OjpVcmkjaHR0cDovLzE5Mi4xNjEuMTQ3LjE6ODA="],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/lib/checks/zimbra.rb

@@ -0,0 +1,24 @@
+module Intrigue
+module Ident
+module Check
+    class Zimbra < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "service",
+            :vendor =>"Zimbra",
+            :product =>"Server",
+            :match_details =>"login page for zimbra",
+            :match_type => :content_body,
+            :match_content =>  /<title>Zimbra Web Client Sign In/i,
+            :examples => ["https://219.84.198.177:443"],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/lib/checks/zscaler.rb

@@ -0,0 +1,28 @@
+module Intrigue
+module Ident
+module Check
+    class Zscaler < Intrigue::Ident::Check::Base
+
+      def generate_checks(url)
+        [
+          {
+            :type => "service",
+            :vendor =>"Zscaler",
+            :product =>"Zscaler",
+            :match_details =>"server header for Zscaler",
+            :references => ["https://help.zscaler.com/zia/about-private-zens"],
+            :match_type => :content_headers,
+            :match_content =>  /server: Zscaler/i,
+            :dynamic_version => lambda { |x|
+              _first_header_capture(x,/server: Zscaler\/(.*)/i)
+            },
+            :examples => ["http://152.26.176.12:80"],
+            :paths => ["#{url}"]
+          }
+        ]
+      end
+
+    end
+  end
+  end
+  end

data/{ident.rb → lib/intrigue-ident.rb}

@@ -3,15 +3,18 @@ require 'net/http'
 require 'openssl'
 require 'zlib'
 
-require_relative 'lib/check_factory'
-require_relative 'lib/checks/base'
-check_folder = File.expand_path('lib/checks', File.dirname(__FILE__)) # get absolute directory
+require_relative 'check_factory'
+require_relative 'checks/base'
+check_folder = File.expand_path('checks', File.dirname(__FILE__)) # get absolute directory
 Dir["#{check_folder}/*.rb"].each { |file| require_relative file }
 
+require_relative 'traverse_exceptions'
+include Intrigue::Ident::TraverseExceptions
+
 module Intrigue
   module Ident
 
-    VERSION=0.40
+    VERSION=0.60
 
     def generate_requests_and_check(url)
 
@@ -51,7 +54,7 @@ module Intrigue
     results.compact
     end
 
-    def check_intrigue_uri(intrigue_uri_data)
+    def check_intrigue_uri_hash(intrigue_uri_data)
 
       results = []
 
@@ -70,7 +73,7 @@ module Intrigue
 
         # call each check, collecting the product if it's a match
         ggc.last.each do |check|
-          results << _match_uri(check, intrigue_uri_data)
+          results << _match_uri_hash(check, intrigue_uri_data)
         end
       end
 
@@ -78,19 +81,48 @@ module Intrigue
     results.compact
     end
 
+    # remove bad checks we need to roll back
+    def remove_bad_ident_matches(matches)
+      passed_matches = []
+      matches.each do |m|
+        next if (m["match_type"] == "content_body" &&
+                        m["matched_content"] == "(?-mix:Drupal)")
+        passed_matches << m
+      end
+    passed_matches
+    end
+
     private
 
     def _construct_match_response(check, data)
-      {
-        :version => (check[:dynamic_version].call(data) if check[:dynamic_version]) || check[:version],
-        :name => check[:name],
-        :tags => check[:tags],
-        :match => check[:type],
-        :hide => check[:hide]
-      }
+      calculated_version = (check[:dynamic_version].call(data) if check[:dynamic_version]) || check[:version]
+
+      calculated_type = "a" if check[:type] == "application"
+      calculated_type = "h" if check[:type] == "hardware"
+      calculated_type = "o" if check[:type] == "operating_system"
+      calculated_type = "s" if check[:type] == "service" # literally made up
+
+      cpe_string = "cpe:/#{calculated_type}:#{check[:vendor]}:#{check[:product]}".downcase
+      cpe_string << ":#{calculated_version}".downcase if calculated_version
+
+    {
+      "type" => check[:type],
+      "vendor" => check[:vendor],
+      "product" => check[:product],
+      "version" => calculated_version,
+      "tags" => check[:tags],
+      "matched_content" => check[:match_content],
+      "match_type" => check[:match_type],
+      "match_details" => check[:match_details],
+      "hide" => check[:hide],
+      "cpe" => cpe_string
+    }
     end
 
-    def _match_uri(check,data)
+    def _match_uri_hash(check,data)
+      return nil unless check && data
+
+      #puts "Trying to match #{check[:vendor]} #{check[:product]}: #{data["details"]["cookies"][0..10]}"
 
       # data[:body] => page body
       # data[:headers] => block of text with headers, one per line
@@ -98,16 +130,23 @@ module Intrigue
       # data[:body_md5] => md5 hash of the body
       # if type "content", do the content check
 
-
-      if check[:type] == :content_body
-        match = _construct_match_response(check,data) if data["details"]["hidden_response_data"] =~ check[:content]
-      elsif check[:type] == :content_headers
-        match = _construct_match_response(check,data) if data["details"]["headers"].join("\n") =~ check[:content]
-      elsif check[:type] == :content_cookies
+      if check[:match_type] == :content_body
+        if data["details"] && data["details"]["hidden_response_data"]
+          match = _construct_match_response(check,data) if data["details"]["hidden_response_data"] =~ check[:match_content]
+        end
+      elsif check[:match_type] == :content_headers
+        if data["details"] && data["details"]["headers"]
+          match = _construct_match_response(check,data) if data["details"]["headers"].join("\n") =~ check[:match_content]
+        end
+      elsif check[:match_type] == :content_cookies
         # Check only the set-cookie header
-        match = _construct_match_response(check,data) if data["details"]["cookies"] =~ check[:content]
-      elsif check[:type] == :checksum_body
-        match = _construct_match_response(check,data) if Digest::MD5.hexdigest(data["details"]["response_data_hash"]) == check[:checksum]
+        if data["details"] && data["details"]["cookies"]
+          match = _construct_match_response(check,data) if data["details"]["cookies"] =~ check[:match_content]
+        end
+      elsif check[:match_type] == :checksum_body
+        if data["details"] && data["details"]["response_data_hash"]
+          match = _construct_match_response(check,data) if Digest::MD5.hexdigest(data["details"]["response_data_hash"]) == check[:checksum]
+        end
       end
 
     match
@@ -130,7 +169,7 @@ module Intrigue
       	"hidden": false,
       	"detail_string": "Server:  | App:  | Title: Index page",
       	"details": {
-      		"uri": "http://69.162.37.69:80",
+      		"uri": "http://69.112.37.69:80",
       		"code": "200",
       		"port": 80,
       		"forms": false,
@@ -140,8 +179,9 @@ module Intrigue
       		"host_id": 1571,
       		"scripts": [],
       		"products": [],
+          "cookies": "",
       		"protocol": "tcp",
-      		"ip_address": "69.162.37.69",
+      		"ip_address": "69.112.37.69",
       		"javascript": [],
       		"fingerprint": [],
       		"api_endpoint": false,
@@ -156,12 +196,6 @@ module Intrigue
       		"hidden_response_data": "",
       		"hidden_screenshot_contents": """
       	},
-      	"task_results": [{
-      		"id": 32,
-      		"name": "masscan_scan_on_69.162.0.0/18",
-      		"base_entity_name": "69.162.0.0/18",
-      		"base_entity_type": "Intrigue::Entity::NetBlock"
-      	}],
       	"generated_at": "2018-07-04T03:43:11+00:00"
       }'
 =end
@@ -178,7 +212,7 @@ module Intrigue
       data["details"]["response_data_hash"] = Digest::SHA256.base64digest("#{response.body}")
 
       # call the actual matcher & return
-      _match_uri check, data
+      _match_uri_hash check, data
     end
 
     def _http_request(method, uri_string, credentials=nil, headers={}, data=nil, limit = 10, open_timeout=15, read_timeout=15)