recog 2.3.15 → 2.3.20

data/lib/recog/version.rb

@@ -1,3 +1,3 @@
 module Recog
-  VERSION = '2.3.15'
+  VERSION = '2.3.20'
 end

data/requirements.txt

@@ -1,2 +1,2 @@
-lxml==4.5.1
+lxml==4.6.3
 pyyaml

data/update_cpes.py

@@ -1,28 +1,37 @@
 #!/usr/bin/env python
 
-import yaml
 import logging
 import re
 import sys
 
+import yaml
 from lxml import etree
 
 def parse_r7_remapping(file):
     with open(file) as remap_file:
-        return yaml.load(remap_file)["mappings"]
+        return yaml.safe_load(remap_file)["mappings"]
 
 def parse_cpe_vp_map(file):
     vp_map = {} # cpe_type -> vendor -> products
     parser = etree.XMLParser(remove_comments=False)
     doc = etree.parse(file, parser)
     namespaces = {'ns': 'http://cpe.mitre.org/dictionary/2.0', 'meta': 'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2'}
-    for cpe_name in doc.xpath("//ns:cpe-list/ns:cpe-item/@name", namespaces=namespaces):
+    for entry in doc.xpath("//ns:cpe-list/ns:cpe-item", namespaces=namespaces):
+        cpe_name = entry.get("name")
+        if not cpe_name:
+            continue
+
+        # If the entry is deprecated then don't add it to our list of valid CPEs.
+        if entry.get("deprecated"):
+            continue
+
         cpe_match = re.match('^cpe:/([aho]):([^:]+):([^:]+)', cpe_name)
+
         if cpe_match:
             cpe_type, vendor, product = cpe_match.group(1, 2, 3)
-            if not cpe_type in vp_map:
+            if cpe_type not in vp_map:
                 vp_map[cpe_type] = {}
-            if not vendor in vp_map[cpe_type]:
+            if vendor not in vp_map[cpe_type]:
                 vp_map[cpe_type][vendor] = set()
             product = product.replace('%2f', '/')
             vp_map[cpe_type][vendor].add(product)
@@ -34,12 +43,12 @@ def parse_cpe_vp_map(file):
 def main():
     if len(sys.argv) != 4:
         logging.critical("Expecting exactly 3 arguments; recog XML file, CPE 2.3 XML dictionary, JSON remapping, got %s", (len(sys.argv) - 1))
-        exit(1)
+        sys.exit(1)
 
     cpe_vp_map = parse_cpe_vp_map(sys.argv[2])
     if not cpe_vp_map:
         logging.critical("No CPE vendor => product mappings read from CPE 2.3 XML dictionary %s", sys.argv[2])
-        exit(1)
+        sys.exit(1)
 
     r7_vp_map = parse_r7_remapping(sys.argv[3])
     if not r7_vp_map:
@@ -47,6 +56,86 @@ def main():
 
     update_cpes(sys.argv[1], cpe_vp_map, r7_vp_map)
 
+def lookup_cpe(vendor, product, cpe_type, cpe_table, remap):
+    """Identify the correct vendor and product values for a CPE
+
+    This function attempts to determine the correct CPE using vendor and product
+    values supplied by the caller as well as a remapping dictionary for mapping
+    these values to more correct values used by NIST.
+
+    For example, the remapping might tell us that a value of 'alpine' for the
+    vendor string should be 'aplinelinux' instead, or for product 'solaris'
+    should be 'sunos'.
+
+    This function should only emit values seen in the official NIST CPE list
+    which is provided to it in cpe_table.
+
+    Lookup priority:
+    1. Original vendor / product
+    2. Original vendor / remap product
+    3. Remap vendor / original product
+    4. Remap vendor / remap product
+
+    Args:
+        vendor (str):  vendor name
+        product (str): product name
+        cpe_type (str): CPE type - o, a, h, etc.
+        cpe_table (dict): dict containing the official NIST CPE data
+        remap (dict): dict containing the remapping values
+    Returns:
+        success, vendor, product
+    """
+
+    if (
+        vendor in cpe_table[cpe_type]
+        and product in cpe_table[cpe_type][vendor]
+    ):
+        # Hot path, success with original values
+        return True, vendor, product
+
+    # Everything else depends on a remap of some sort.
+    # get the remappings for this one vendor string.
+    vendor_remap = None
+
+    remap_type = remap.get(cpe_type, None)
+    if remap_type:
+        vendor_remap = remap_type.get(vendor, None)
+
+    if vendor_remap:
+        # If we have product remappings, work that angle next
+        possible_product = None
+        if (
+            vendor_remap.get('products', None)
+            and product in vendor_remap['products']
+        ):
+            possible_product = vendor_remap['products'][product]
+
+        if (vendor in cpe_table[cpe_type]
+            and possible_product
+            and possible_product in cpe_table[cpe_type][vendor]):
+            # Found original vendor, remap product
+            return True, vendor, possible_product
+
+        # Start working the process to find a match with a remapped vendor name
+        if vendor_remap.get('vendor', None):
+            new_vendor = vendor_remap['vendor']
+
+            if new_vendor in cpe_table[cpe_type]:
+
+                if product in cpe_table[cpe_type][new_vendor]:
+                    # Found remap vendor, original product
+                    return True, new_vendor, product
+
+                if possible_product and possible_product in cpe_table[cpe_type][new_vendor]:
+                    # Found remap vendor, remap product
+                    return True, new_vendor, possible_product
+
+
+    logging.error("Product %s from vendor %s invalid for CPE %s and no mapping",
+                  product, vendor, cpe_type)
+    return False, None, None
+
+
 def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
     parser = etree.XMLParser(remove_comments=False, remove_blank_text=True)
     doc = etree.parse(xml_file, parser)
@@ -114,55 +203,27 @@ def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
                     continue
 
                 vendor = vendor.lower().replace(' ', '_').replace(',', '')
-                product = product.lower().replace(' ', '_').replace(',', '')
+                product = product.lower().replace(' ', '_').replace(',', '').replace('!', '%21')
                 if 'unknown' in [vendor, product]:
                     continue
 
                 if (vendor.startswith('{') and vendor.endswith('}')) or (product.startswith('{') and product.endswith('}')):
                     continue
 
-                remapped_vendor = False
-                og_vendor = vendor
-                if not vendor in cpe_vp_map[cpe_type]:
-                    if vendor in r7_vp_map:
-                        vendor = r7_vp_map[vendor]['vendor']
-                        remapped_vendor = True
-                        if not vendor in cpe_vp_map[cpe_type]:
-                            logging.error("Remapped vendor %s (remapped from %s) invalid for CPE %s (product %s)", vendor, og_vendor, cpe_type, product)
-                            continue
-                    else:
-                        logging.error("Vendor %s invalid for CPE %s and no remapping (product %s)", vendor, cpe_type, product)
-                        continue
-
-
-                # if the product as specified is not found in the CPE dictionary for this vendor
-                if not product in cpe_vp_map[cpe_type][vendor]:
-                    # if this vendor has a remapping from R7
-                    if og_vendor in r7_vp_map and 'products' in r7_vp_map[og_vendor]:
-                        # if this product has a remapping for this vendor from R7
-                        if product in r7_vp_map[og_vendor]['products']:
-                            og_product = product
-                            product = r7_vp_map[og_vendor]['products'][product]
-                            # ensure that the remapped product is valid for the given vendor in CPE
-                            if not product in cpe_vp_map[cpe_type][vendor]:
-                                logging.error("Remapped product %s (remapped from %s) from vendor %s invalid for CPE %s", product, og_product, vendor, cpe_type)
-                                continue
-                        else:
-                            if remapped_vendor:
-                                logging.error("Product %s from vendor %s (remapped from %s) invalid for CPE %s and no mapping", product, vendor, og_vendor, cpe_type)
-                            else:
-                                logging.error("Product %s from vendor %s invalid for CPE %s and no mapping", product, vendor, cpe_type)
-                            continue
-                    else:
-                        if remapped_vendor:
-                            logging.error("Vendor %s (remapped from %s) is valid for CPE %s but product %s not valid and no mapping", vendor, og_vendor, cpe_type, product)
-                        else:
-                            logging.error("Vendor %s is valid for CPE %s but product %s not valid and no mapping", vendor, cpe_type, product)
-                        continue
+                success, vendor, product = lookup_cpe(vendor, product, cpe_type, cpe_vp_map, r7_vp_map)
+                if not success:
+                    continue
+
+                # Sanity check the value to ensure that no invalid values will
+                # slip in due to logic or mapping bugs.
+                # If it's not in the official NIST list then log it and kick it out
+                if product not in cpe_vp_map[cpe_type][vendor]:
+                    logging.error("Invalid CPE type %s created for vendor %s and product %s. This may be due to an invalid mapping.", cpe_type, vendor, product)
+                    continue
 
                 # building the CPE string
-                # Last minute escaping of '/'
-                product = product.replace('/', '\/')
+                # Last minute escaping of '/' and `!`
+                product = product.replace('/', '\/').replace('%21', '\!')
                 cpe_value = 'cpe:/{}:{}:{}'.format(cpe_type, vendor, product)
 
                 if version:
@@ -185,5 +246,5 @@ def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
         xml_out.write(etree.tostring(root, pretty_print=True, xml_declaration=True, encoding=doc.docinfo.encoding))
 
 if __name__ == '__main__':
-    try: exit(main())
+    try: sys.exit(main())
     except KeyboardInterrupt: pass

data/xml/apache_modules.xml

@@ -220,6 +220,36 @@
     <param pos="0" name="service.component.product" value="mod_auth_ldap"/>
   </fingerprint>
 
+  <fingerprint pattern="mod_auth_oracle/(\S+)$">
+    <description>mod_auth_oracle with version</description>
+    <example service.component.version="1.2.3">mod_auth_oracle/1.2.3</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_auth_oracle"/>
+    <param pos="1" name="service.component.version"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_auth_oracle/?$">
+    <description>mod_auth_oracle without version</description>
+    <example>mod_auth_oracle/</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_auth_oracle"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_auth_pgsql/(\S+)$">
+    <description>mod_auth_pgsql with version</description>
+    <example service.component.version="1.2.3">mod_auth_pgsql/1.2.3</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_auth_pgsql"/>
+    <param pos="1" name="service.component.version"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_auth_pgsql/?$">
+    <description>mod_auth_pgsql without version</description>
+    <example>mod_auth_pgsql/</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_auth_pgsql"/>
+  </fingerprint>
+
   <fingerprint pattern="mod_auth_radius/(\S+)$">
     <description>mod_auth_radius with version</description>
     <example service.component.version="1.2.3">mod_auth_radius/1.2.3</example>
@@ -978,6 +1008,36 @@
     <param pos="0" name="service.component.product" value="mod_filter"/>
   </fingerprint>
 
+  <fingerprint pattern="mod_frontpage/(\S+)$">
+    <description>mod_frontpage with version</description>
+    <example service.component.version="1.2.3">mod_frontpage/1.2.3</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_frontpage"/>
+    <param pos="1" name="service.component.version"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_frontpage/?$">
+    <description>mod_frontpage without version</description>
+    <example>mod_frontpage/</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_frontpage"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_gzip/(\S+)$">
+    <description>mod_gzip with version</description>
+    <example service.component.version="1.2.3">mod_gzip/1.2.3</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_gzip"/>
+    <param pos="1" name="service.component.version"/>
+  </fingerprint>
+
+  <fingerprint pattern="mod_gzip/?$">
+    <description>mod_gzip without version</description>
+    <example>mod_gzip/</example>
+    <param pos="0" name="service.component.vendor" value="Apache"/>
+    <param pos="0" name="service.component.product" value="mod_gzip"/>
+  </fingerprint>
+
   <fingerprint pattern="mod_headers/(\S+)$">
     <description>mod_headers with version</description>
     <example service.component.version="1.2.3">mod_headers/1.2.3</example>

data/xml/dns_versionbind.xml

@@ -427,7 +427,7 @@
     <param pos="0" name="service.cpe23" value="cpe:/a:powerdns:authoritative_server:{service.version}"/>
   </fingerprint>
 
-  <fingerprint pattern="^PowerDNS Authoritative Server (\d\.[\w.]+(?:-rc\d)?(?:-alpha\d)?(?:-beta\d)?[^ ]*) \(built [\w\s:]+ by [\w]+\@[\w.-:-]*\)$">
+  <fingerprint pattern="^PowerDNS Authoritative Server (\d\.[\w.]+(?:-rc\d)?(?:-alpha\d)?(?:-beta\d)?[^ ]*) \(built [\w\s:]+ by [\w]+\@[\w.:-]*\)$">
     <description>PowerDNS Authoritative Server: format 2</description>
     <example service.version="4.0.4">PowerDNS Authoritative Server 4.0.4 (built Jul 26 2017 15:04:27 by root@FreeBSD:11:amd64-default-job-03)</example>
     <example service.version="4.0.0-rc2">PowerDNS Authoritative Server 4.0.0-rc2 (built Jul  4 2016 15:44:39 by root@foo-bar.baz)</example>
@@ -619,17 +619,18 @@
        dnscmd /config /EnableVersionQuery 1
   -->
 
-  <fingerprint pattern="^Microsoft DNS (10.0.\d+)(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS (10.0.\d+)(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2016: GA</description>
     <!-- Windows 10 / 2016 moved towards a rolling release so capturing build
          is required unlike other Windows versions where we use a fixed string.
     -->
 
-    <example service.version="10.0.14393" os.build="10.0.14393">Microsoft DNS 10.0.14393 (383900CE)</example>
+    <example service.version="10.0.14393" os.build="10.0.14393" service.version.version="383900CE">Microsoft DNS 10.0.14393 (383900CE)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="1" name="service.version"/>
+    <param pos="2" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2016"/>
@@ -637,13 +638,14 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2016:-"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.3.9600(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.3.9600(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2012 R2</description>
-    <example>Microsoft DNS 6.3.9600 (25804825)</example>
+    <example service.version.version="25804825">Microsoft DNS 6.3.9600 (25804825)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.3.9600"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2012 R2"/>
@@ -651,13 +653,14 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2012:-"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.2.9200(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.2.9200(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2012</description>
-    <example>Microsoft DNS 6.2.9200 (23F04000)</example>
+    <example service.version.version="23F04000">Microsoft DNS 6.2.9200 (23F04000)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.2.9200"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2012"/>
@@ -665,14 +668,15 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2012:-"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.1.7601(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.1.7601(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2008 R2 Service Pack 1</description>
-    <example>Microsoft DNS 6.1.7601 (1DB15CD4)</example>
+    <example service.version.version="1DB15CD4">Microsoft DNS 6.1.7601 (1DB15CD4)</example>
     <example>Microsoft DNS 6.1.7601</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.1.7601"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2008 R2"/>
@@ -681,13 +685,14 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2008:Service Pack 1"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.1.7600(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.1.7600(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2008 R2</description>
-    <example>Microsoft DNS 6.1.7600 (1DB04228)</example>
+    <example service.version.version="1DB04228">Microsoft DNS 6.1.7600 (1DB04228)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.1.7600"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2008 R2"/>
@@ -708,13 +713,14 @@
     <example>Microsoft DNS 6.0.6100 (2AEF76E)</example>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.0.6003(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.0.6003(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2008 Service Pack 2 - Preview Rollup KB4489887 and later</description>
-    <example>Microsoft DNS 6.0.6003 (1773501D)</example>
+    <example service.version.version="1773501D">Microsoft DNS 6.0.6003 (1773501D)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.0.6003"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2008"/>
@@ -723,13 +729,14 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2008:Service Pack 2"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.0.6002(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.0.6002(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2008 Service Pack 2</description>
-    <example>Microsoft DNS 6.0.6002 (17724D35)</example>
+    <example service.version.version="17724D35">Microsoft DNS 6.0.6002 (17724D35)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.0.6002"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2008"/>
@@ -738,13 +745,14 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2008:Service Pack 2"/>
   </fingerprint>
 
-  <fingerprint pattern="^Microsoft DNS 6.0.6001(?: \(\w+\))?$">
+  <fingerprint pattern="^Microsoft DNS 6.0.6001(?: \(([^)]+)\))?$">
     <description>Microsoft DNS on Windows 2008 Service Pack 1</description>
-    <example>Microsoft DNS 6.0.6001 (17714726)</example>
+    <example service.version.version="17714726">Microsoft DNS 6.0.6001 (17714726)</example>
     <param pos="0" name="service.vendor" value="Microsoft"/>
     <param pos="0" name="service.family" value="DNS"/>
     <param pos="0" name="service.product" value="DNS"/>
     <param pos="0" name="service.version" value="6.0.6001"/>
+    <param pos="1" name="service.version.version"/>
     <param pos="0" name="os.vendor" value="Microsoft"/>
     <param pos="0" name="os.family" value="Windows"/>
     <param pos="0" name="os.product" value="Windows Server 2008"/>
@@ -753,6 +761,21 @@
     <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2008:Service Pack 1"/>
   </fingerprint>
 
+  <fingerprint pattern="^Microsoft DNS 5.2.3790(?: \(([^)]+)\))?$">
+    <description>Microsoft DNS on Windows 2003</description>
+    <example service.version.version="ECE135D">Microsoft DNS 5.2.3790 (ECE135D)</example>
+    <param pos="0" name="service.vendor" value="Microsoft"/>
+    <param pos="0" name="service.family" value="DNS"/>
+    <param pos="0" name="service.product" value="DNS"/>
+    <param pos="0" name="service.version" value="5.2.3790"/>
+    <param pos="1" name="service.version.version"/>
+    <param pos="0" name="os.vendor" value="Microsoft"/>
+    <param pos="0" name="os.family" value="Windows"/>
+    <param pos="0" name="os.product" value="Windows Server 2003"/>
+    <param pos="0" name="os.build" value="5.2.3790"/>
+    <param pos="0" name="os.cpe23" value="cpe:/o:microsoft:windows_server_2003:-"/>
+  </fingerprint>
+
   <fingerprint pattern="^DNSServer$">
     <description>Synology DNS service</description>
     <example>DNSServer</example>