recog 2.3.23 → 3.0.1

data/misc/convert_mysql_err

@@ -1,61 +0,0 @@
-#!/usr/bin/env ruby
-
-# Takes the MySQL error messages from sql/share/errmsg-utf8.txt, locates the
-# provided error message type (for example, ER_HOST_NOT_PRIVILEGED), then
-# creates XML snippets for each locale to be used in Recog.  Note that this
-# cannot be used as-is to generate mysql_errors.xml, or oftentimes even parts
-# -- it merely spits out XML snippets that you can start with; many will still
-# need to be modified by hand.
-
-require 'builder'
-require 'open-uri'
-require 'securerandom'
-
-def generate_recog(error_name, locale, error_message)
-  xml = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
-  xml.fingerprint(pattern: error_message) do
-    xml.description "Oracle MySQL error #{error_name} (#{locale})"
-    xml.example(error_message)
-    xml.param(pos: 0, name: 'service.vendor', value: 'Oracle')
-    xml.param(pos: 0, name: 'service.family', value: 'MySQL')
-    xml.param(pos: 0, name: 'service.product', value: 'MySQL')
-  end
-end
-
-unless ARGV.size == 2
-  fail "Usage: #{$PROGRAM_NAME} <path/URI for errmsg-utf8.txt> <error name>"
-end
-
-path = ARGV.first
-error_name = ARGV.last
-
-lines = IO.readlines(open(path))
-
-fail "Nothing read from #{path}" if lines.empty?
-
-unless (error_start = lines.find_index { |line| line.strip =~ /^#{error_name}(?:\s+\S+)?$/ })
-  fail "Unable to find #{error_name} in #{path}"
-end
-
-locale_map = {}
-lines.slice(error_start + 1, lines.size).each do |line|
-  if /^\s+(?<locale>\S+)\s+"(?<error_message>.*)",?$/ =~ line
-    locale_map[locale] = error_message
-  else
-    break
-  end
-end
-
-# Many of the error messages contain format strings.  This can be problematic
-# in that they need to be removed or otherwise handled as part of the 'pattern'
-# attribute and appropriately filled in in any example elements.  So simply try
-# a rough count of the possible format strings and warn the user so that they
-# can deal with it.
-format_count = locale_map.values.map { |error_message| error_message.scan(/%/).size }.inject(&:+)
-unless format_count == 0
-  warn("#{format_count} possible format strings found -- you'll need to deal with this")
-end
-
-Hash[locale_map.sort].map do |locale, error_message|
-  generate_recog(error_name, locale, error_message)
-end

data/misc/order.xsl

@@ -1,17 +0,0 @@
-<?xml version="1.0"?>
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
-  <xsl:output encoding="UTF-8" indent="yes" method="xml"/>
-  <xsl:template match="@*|node()">
-    <xsl:copy>
-      <xsl:apply-templates select="@*|node()"/>
-    </xsl:copy>
-  </xsl:template>
-  <xsl:template match="fingerprints/fingerprint">
-    <xsl:copy>
-      <xsl:copy-of select="@*"/>
-      <xsl:apply-templates select="description"/>
-      <xsl:apply-templates select="example"/>
-      <xsl:apply-templates select="param"/>
-    </xsl:copy>
-  </xsl:template>
-</xsl:stylesheet>

data/requirements.txt

@@ -1,2 +0,0 @@
-lxml==4.6.5
-pyyaml

data/spec/lib/fingerprint_self_test_spec.rb

@@ -1,175 +0,0 @@
-require 'recog/db'
-require 'regexp_parser'
-require 'nokogiri'
-
-describe Recog::DB do
-  let(:schema) { Nokogiri::XML::Schema(open(File.expand_path(File.join(%w(xml fingerprints.xsd))))) }
-  Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
-
-    describe "##{File.basename(xml_file_name)}" do
-
-      it "is valid XML" do
-        doc = Nokogiri::XML(open(xml_file_name))
-        errors = schema.validate(doc)
-        expect(errors).to be_empty, "#{xml_file_name} is invalid recog XML -- #{errors.inspect}"
-      end
-
-      db = Recog::DB.new(xml_file_name)
-
-      it "has a match key" do
-        expect(db.match_key).not_to be_nil
-        expect(db.match_key).not_to be_empty
-      end
-
-      it "has valid 'preference' value" do
-          # Reserve values below 0.10 and above 0.90 for users
-          # See xml/fingerprints.xsd
-          expect(db.preference.class).to be ::Float
-          expect(db.preference).to be_between(0.10, 0.90)
-      end
-
-      fp_descriptions = []
-      db.fingerprints.each_index do |i|
-        fp = db.fingerprints[i]
-
-        it "doesn't have a duplicate description" do
-          if fp_descriptions.include?(fp.name)
-            fail "'#{fp.name}'s description is not unique"
-          else
-            fp_descriptions << fp.name
-          end
-        end
-
-        context "#{fp.name}" do
-          param_names = []
-          it "has consistent os.device and hw.device" do
-            if fp.params['os.device'] && fp.params['hw.device'] && (fp.params['os.device'] != fp.params['hw.device'])
-              fail "#{fp.name} has both hw.device and os.device but with differing values"
-            end
-          end
-          fp.params.each do |param_name, pos_value|
-            pos, value = pos_value
-            it "has valid looking fingerprint parameter names" do
-              unless param_name =~ /^(?:cookie|[^\.]+\..*)$/
-                fail "'#{param_name}' is invalid"
-              end
-            end
-
-            it "doesn't have param values for capture params" do
-              if pos > 0 && !value.to_s.empty?
-                fail "'#{fp.name}'s #{param_name} is a non-zero pos but specifies a value of '#{value}'"
-              end
-            end
-
-            it "has parameter values other than General, Server or Unknown, which are not helpful" do
-              if pos == 0 && value =~ /^(?i:general|server|unknown)$/
-                fail "'#{param_name}' has general/server/unknown value '#{value}'"
-              end
-            end
-
-            it "doesn't omit values for non-capture params" do
-              if pos == 0 && value.to_s.empty?
-                fail "'#{fp.name}'s #{param_name} is not a capture (pos=0) but doesn't specify a value"
-              end
-            end
-
-            it "doesn't have duplicate params" do
-              if param_names.include?(param_name)
-                fail "'#{fp.name}'s has duplicate #{param_name}"
-              else
-                param_names << param_name
-              end
-            end
-
-            it "uses interpolation correctly" do
-              if pos == 0 && /\{(?<interpolated>[^\s{}]+)\}/ =~ value
-                unless fp.params.key?(interpolated)
-                  fail "'#{fp.name}' uses interpolated value '#{interpolated}' that does not exist"
-                end
-              end
-            end
-          end
-        end
-
-        context "#{fp.regex}" do
-
-          it "has a valid looking name" do
-            expect(fp.name).not_to be_nil
-            expect(fp.name).not_to be_empty
-          end
-
-          it "has a regex" do
-            expect(fp.regex).not_to be_nil
-            expect(fp.regex.class).to be ::Regexp
-          end
-
-          it 'uses capturing regular expressions properly' do
-            # the list of index-based captures that the fingerprint is expecting
-            expected_capture_positions = fp.params.values.map(&:first).map(&:to_i).select { |position| position > 0 }
-            if fp.params.empty? && expected_capture_positions.size > 0
-              fail "Non-asserting fingerprint with regex #{fp.regex} captures #{expected_capture_positions.size} time(s); 0 are needed"
-            else
-              # parse the regex and count the number of captures
-              actual_capture_positions = []
-              capture_number = 1
-              Regexp::Scanner.scan(fp.regex).each do |token_parts|
-                if token_parts.first == :group  && ![:close, :passive, :options, :options_switch].include?(token_parts[1])
-                  actual_capture_positions << capture_number
-                  capture_number += 1
-                end
-              end
-              # compare the captures actually performed to those being used and ensure that they contain
-              # the same elements regardless of order, preventing, over-, under- and other forms of mis-capturing.
-              actual_capture_positions = actual_capture_positions.sort.uniq
-              expected_capture_positions = expected_capture_positions.sort.uniq
-              expect(actual_capture_positions).to eq(expected_capture_positions),
-                "Regex has #{actual_capture_positions.size} capture groups, but the fingerprint expected #{expected_capture_positions.size} extractions."
-            end
-          end
-
-          # Not yet enforced
-          # it "has test cases" do
-          #  expect(fp.tests.length).not_to equal(0)
-          # end
-
-          it "Has a reasonable number (<= 20) of test cases" do
-            expect(fp.tests.length).to be <= 20
-          end
-
-          fp_examples = []
-          fp.tests.each do |example|
-            it "doesn't have a duplicate examples" do
-              if fp_examples.include?(example.content)
-                fail "'#{fp.name}' has duplicate example '#{example.content}'"
-              else
-                fp_examples << example.content
-              end
-            end
-            it "Example '#{example.content}' matches this regex" do
-              match = fp.match(example.content)
-              expect(match).to_not be_nil, 'Regex did not match'
-              # test any extractions specified in the example
-              example.attributes.each_pair do |k,v|
-                next if k == '_encoding'
-                next if k == '_filename'
-                expect(match[k]).to eq(v), "Regex didn't extract expected value for fingerprint attribute #{k} -- got #{match[k]} instead of #{v}"
-              end
-            end
-
-            it "Example '#{example.content}' matches this regex first" do
-              db.fingerprints.slice(0, i).each_index do |previous_i|
-                prev_fp = db.fingerprints[previous_i]
-                prev_fp.tests.each do |prev_example|
-                  match = prev_fp.match(example.content)
-                  expect(match).to be_nil, "Matched regex ##{previous_i} (#{db.fingerprints[previous_i].regex}) rather than ##{i} (#{db.fingerprints[i].regex})"
-                end
-              end
-            end
-          end
-
-        end
-      end
-
-    end
-  end
-end

data/tools/dev/hooks/pre-commit

@@ -1,21 +0,0 @@
-#!/bin/sh
-#
-# Hook script to verify changes about to be committed.
-# The hook should exit with non-zero status after issuing an appropriate
-# message if it wants to stop the commit.
-
-# Verify that each fingerprint asserts known identifiers.
-git diff --cached --name-only --diff-filter=ACM -z xml/*.xml | xargs -0 ./bin/recog_standardize --write
-
-# get status
-status=$?
-
-if [ $status -ne 0 ]; then
-    echo "Please review any new additions to the text files under 'identifiers/'."
-    echo "If any of these names are close to an existing name, update the offending"
-    echo "fingerprint to use the existing name instead. Once the fingerprints are fixed,"
-    echo "remove the 'extra' names from the identifiers files, and run the tool again."
-    exit 1
-fi
-
-exit 0

data/update_cpes.py

@@ -1,343 +0,0 @@
-#!/usr/bin/env python
-
-import logging
-import re
-import sys
-
-import yaml
-from lxml import etree
-
-BASE_LOG_FORMAT = '%(levelname)s: %(message)s'
-
-# CPE w/o 2.3 component: cpe:/a:nginx:nginx:0.1.0"
-REGEX_CPE = re.compile('^cpe:/([aho]):([^:]+):([^:]+)')
-# CPE w/  2.3 component: cpe:2.3:a:f5:nginx:0.1.0:*:*:*:*:*:*:*
-REGEX_CPE_23 = re.compile('^cpe:2.3:([aho]):([^:]+):([^:]+)')
-
-XML_PATH_DEPRECATED_BY = "./{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item/{http://scap.nist.gov/schema/cpe-extension/2.3}deprecation/{http://scap.nist.gov/schema/cpe-extension/2.3}deprecated-by"
-
-
-def parse_r7_remapping(file):
-    with open(file) as remap_file:
-        return yaml.safe_load(remap_file)["mappings"]
-
-
-def update_vp_map(target_map, cpe_type, vendor, product):
-    """Add an entry to the dict tracking valid combinations
-    """
-
-    if cpe_type not in target_map:
-        target_map[cpe_type] = {}
-
-    if vendor not in target_map[cpe_type]:
-        target_map[cpe_type][vendor] = set()
-
-    product = product.replace('%2f', '/')
-    target_map[cpe_type][vendor].add(product)
-
-
-def update_deprecated_map(target_map, dep_string, entry):
-    """Add an entry to the dict tracking deprecations
-
-    target_map example:
-
-    {
-      "a:100plus:101eip":
-        {
-          "deprecated_date": "2021-06-10T15:28:05.490Z",
-          "deprecated_by": "a:hundredplus:101eip"
-        }
-    }
-
-    Args:
-        target_map (dict): dict containing deprecations
-        dep_string (str): key to add in the format of 'type:vendor:product'
-        entry (lxml.etree._Element): XML element to pull additional data from
-
-    Returns:
-        None, target_map modified in place
-    """
-
-    deprecated_date = entry.get("deprecation_date", "")
-
-    # Find the CPE that deprecated this entry
-    raw_dep_by = entry.find(XML_PATH_DEPRECATED_BY).get('name')
-
-    # Extract the type, vendor, product
-    dep_by_match = REGEX_CPE_23.match(raw_dep_by)
-    if not dep_by_match:
-        logging.error("CPE %s is deprecated but we can't build the deprecation mapping entry for some reason.", dep_string)
-        return
-
-    dep_type, dep_vendor, dep_product = dep_by_match.group(1, 2, 3)
-    deprecated_by = "{}:{}:{}".format(dep_type, dep_vendor, dep_product)
-
-    if dep_string not in target_map:
-        target_map[dep_string] = {}
-
-    if not target_map[dep_string].get('deprecated_date'):
-        target_map[dep_string]['deprecated_date'] = deprecated_date
-
-    if not target_map[dep_string].get('deprecated_by'):
-        target_map[dep_string]['deprecated_by'] = deprecated_by
-
-
-def parse_cpe_vp_map(file):
-    deprecated_map = {}
-    vp_map = {} # cpe_type -> vendor -> products
-
-    parser = etree.XMLParser(remove_comments=False)
-    doc = etree.parse(file, parser)
-    namespaces = {
-        'ns':     'http://cpe.mitre.org/dictionary/2.0',
-        'meta':   'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2'
-    }
-    for entry in doc.xpath("//ns:cpe-list/ns:cpe-item", namespaces=namespaces):
-        cpe_name = entry.get("name")
-        if not cpe_name:
-            continue
-
-        cpe_match = REGEX_CPE.match(cpe_name)
-        if cpe_match:
-            cpe_type, vendor, product = cpe_match.group(1, 2, 3)
-            # If the entry is deprecated then don't add it to our list of valid
-            # CPEs, but instead add it to a list for reference later.
-            if entry.get("deprecated"):
-                # This will be the key under which we store the deprecation data
-                deprecated_string = "{}:{}:{}".format(cpe_type, vendor, product)
-
-                update_deprecated_map(deprecated_map, deprecated_string, entry)
-                continue
-
-            update_vp_map(vp_map, cpe_type, vendor, product)
-
-        else:
-            logging.error("Unexpected CPE %s", cpe_name)
-
-    return vp_map, deprecated_map
-
-
-def lookup_cpe(vendor, product, cpe_type, cpe_table, remap, deprecated_map):
-    """Identify the correct vendor and product values for a CPE
-
-    This function attempts to determine the correct CPE using vendor and product
-    values supplied by the caller as well as a remapping dictionary for mapping
-    these values to more correct values used by NIST.
-
-    For example, the remapping might tell us that a value of 'alpine' for the
-    vendor string should be 'alpinelinux' instead, or for product 'solaris'
-    should be 'sunos'.
-
-    This function should only emit values seen in the official NIST CPE list
-    which is provided to it in cpe_table.
-
-    Lookup priority:
-    1. Original vendor / product
-    2. Original vendor / remap product
-    3. Remap vendor / original product
-    4. Remap vendor / remap product
-
-    Args:
-        vendor (str):  vendor name
-        product (str): product name
-        cpe_type (str): CPE type - o, a, h, etc.
-        cpe_table (dict): dict containing the official NIST CPE data
-        remap (dict): dict containing the remapping values
-        deprecated_cves (set): set of all deprecated CPEs in the format
-            'type:vendor:product'
-    Returns:
-        success, vendor, product
-    """
-
-    if (
-        vendor in cpe_table[cpe_type]
-        and product in cpe_table[cpe_type][vendor]
-    ):
-        # Hot path, success with original values
-        return True, vendor, product
-
-    # Everything else depends on a remap of some sort.
-    # get the remappings for this one vendor string.
-    vendor_remap = None
-
-    remap_type = remap.get(cpe_type, None)
-    if remap_type:
-        vendor_remap = remap_type.get(vendor, None)
-
-    if vendor_remap:
-        # If we have product remappings, work that angle next
-        possible_product = None
-        if (
-            vendor_remap.get('products', None)
-            and product in vendor_remap['products']
-        ):
-            possible_product = vendor_remap['products'][product]
-
-        if (vendor in cpe_table[cpe_type]
-            and possible_product
-            and possible_product in cpe_table[cpe_type][vendor]):
-            # Found original vendor, remap product
-            return True, vendor, possible_product
-
-        # Start working the process to find a match with a remapped vendor name
-        if vendor_remap.get('vendor', None):
-            new_vendor = vendor_remap['vendor']
-
-            if new_vendor in cpe_table[cpe_type]:
-
-                if product in cpe_table[cpe_type][new_vendor]:
-                    # Found remap vendor, original product
-                    return True, new_vendor, product
-
-                if possible_product and possible_product in cpe_table[cpe_type][new_vendor]:
-                    # Found remap vendor, remap product
-                    return True, new_vendor, possible_product
-
-    deprecated_string = "{}:{}:{}".format(cpe_type, vendor, product)
-    if deprecated_map.get(deprecated_string, False):
-        dep_by = deprecated_map[deprecated_string].get("deprecated_by", "")
-        dep_date = deprecated_map[deprecated_string].get("deprecated_date", "")
-        logging.error("Product %s from vendor %s invalid for CPE %s and no mapping.  This combination is DEPRECATED by %s at %s",
-                    product, vendor, cpe_type, dep_by, dep_date)
-    else:
-        logging.error("Product %s from vendor %s invalid for CPE %s and no mapping.",
-                    product, vendor, cpe_type)
-
-    return False, None, None
-
-
-def update_cpes(xml_file, cpe_vp_map, r7_vp_map, deprecated_cves):
-    parser = etree.XMLParser(remove_comments=False, remove_blank_text=True)
-    doc = etree.parse(xml_file, parser)
-
-    for fingerprint in doc.xpath('//fingerprint'):
-
-        # collect all the params, grouping by os and service params that could be used to compute a CPE
-        params = {}
-        for param in fingerprint.xpath('./param'):
-            name = param.attrib['name']
-            # remove any existing CPE params
-            if re.match(r'^.*\.cpe\d{0,2}$', name):
-                param.getparent().remove(param)
-                continue
-
-            match = re.search(r'^(?P<fp_type>hw|os|service(?:\.component)?)\.', name)
-            if match:
-                fp_type = match.group('fp_type')
-                if not fp_type in params:
-                    params[fp_type] = {}
-                if name in params[fp_type]:
-                    raise ValueError('Duplicated fingerprint named {} in fingerprint {} in file {}'.format(name, fingerprint.attrib['pattern'], xml_file))
-                params[fp_type][name] = param
-
-        # for each of the applicable os/service param groups, build a CPE
-        for fp_type in params:
-            if fp_type == 'os':
-                cpe_type = 'o'
-            elif fp_type.startswith('service'):
-                cpe_type = 'a'
-            elif fp_type == 'hw':
-                cpe_type = 'h'
-            else:
-                raise ValueError('Unhandled param type {}'.format(fp_type))
-
-            # extract the vendor/product/version values from each os/service group,
-            # using the static value ('Apache', for example) when pos is 0, and
-            # otherwise use a value that contains interpolation markers such that
-            # products/projects that use recog content can insert the value
-            # extracted from the banner/other data via regex capturing groups
-            fp_data = {
-                'vendor': None,
-                'product': None,
-                'version': '-',
-            }
-            for fp_datum in fp_data:
-                fp_datum_param_name = "{}.{}".format(fp_type, fp_datum)
-                if fp_datum_param_name in params[fp_type]:
-                    fp_datum_e = params[fp_type][fp_datum_param_name]
-                    if fp_datum_e.attrib['pos'] == '0':
-                        fp_data[fp_datum] = fp_datum_e.attrib['value']
-                    else:
-                        fp_data[fp_datum] = "{{{}}}".format(fp_datum_e.attrib['name'])
-
-            vendor = fp_data['vendor']
-            product = fp_data['product']
-            version = fp_data['version']
-
-            # build a reasonable looking CPE value from the vendor/product/version,
-            # lowercasing, replacing whitespace with _, and more
-            if vendor and product:
-                if not cpe_type in cpe_vp_map:
-                    logging.error("Didn't find CPE type '%s' for '%s' '%s'", cpe_type, vendor, product)
-                    continue
-
-                vendor = vendor.lower().replace(' ', '_').replace(',', '')
-                product = product.lower().replace(' ', '_').replace(',', '').replace('!', '%21')
-                if 'unknown' in [vendor, product]:
-                    continue
-
-                if (vendor.startswith('{') and vendor.endswith('}')) or (product.startswith('{') and product.endswith('}')):
-                    continue
-
-                success, vendor, product = lookup_cpe(vendor, product, cpe_type, cpe_vp_map, r7_vp_map, deprecated_cves)
-                if not success:
-                    continue
-
-                # Sanity check the value to ensure that no invalid values will
-                # slip in due to logic or mapping bugs.
-                # If it's not in the official NIST list then log it and kick it out
-                if product not in cpe_vp_map[cpe_type][vendor]:
-                    logging.error("Invalid CPE type %s created for vendor %s and product %s. This may be due to an invalid mapping.", cpe_type, vendor, product)
-                    continue
-
-                # building the CPE string
-                # Last minute escaping of '/' and `!`
-                product = product.replace('/', '\/').replace('%21', '\!')
-                cpe_value = 'cpe:/{}:{}:{}'.format(cpe_type, vendor, product)
-
-                if version:
-                    cpe_value += ":{}".format(version)
-
-                cpe_param = etree.Element('param')
-                cpe_param.attrib['pos'] = '0'
-                cpe_param.attrib['name'] = '{}.cpe23'.format(fp_type)
-                cpe_param.attrib['value'] = cpe_value
-
-                for param_name in params[fp_type]:
-                    param = params[fp_type][param_name]
-                    parent = param.getparent()
-                    index = parent.index(param) + 1
-                    parent.insert(index, cpe_param)
-
-    root = doc.getroot()
-
-    with open(xml_file, 'wb') as xml_out:
-        xml_out.write(etree.tostring(root, pretty_print=True, xml_declaration=True, encoding=doc.docinfo.encoding))
-
-
-def main():
-    if len(sys.argv) != 4:
-        logging.critical("Expecting exactly 3 arguments; recog XML file, CPE 2.3 XML dictionary, JSON remapping, got %s", (len(sys.argv) - 1))
-        sys.exit(1)
-
-    cpe_vp_map, deprecated_map = parse_cpe_vp_map(sys.argv[2])
-    if not cpe_vp_map:
-        logging.critical("No CPE vendor => product mappings read from CPE 2.3 XML dictionary %s", sys.argv[2])
-        sys.exit(1)
-
-    r7_vp_map = parse_r7_remapping(sys.argv[3])
-    if not r7_vp_map:
-        logging.warning("No Rapid7 vendor/product => CPE mapping read from %s", sys.argv[3])
-
-    # update format string for the logging handler to include the recog XML filename
-    logging.basicConfig(force=True, format=f"{sys.argv[1]}: {BASE_LOG_FORMAT}")
-
-    update_cpes(sys.argv[1], cpe_vp_map, r7_vp_map, deprecated_map)
-
-
-if __name__ == '__main__':
-    logging.basicConfig(format=BASE_LOG_FORMAT)
-    try:
-        sys.exit(main())
-    except KeyboardInterrupt:
-        pass