recog 2.3.22 → 3.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -0
- data/LICENSE +1 -1
- data/README.md +25 -16
- data/Rakefile +2 -9
- data/lib/recog/db_manager.rb +1 -1
- data/lib/recog/fingerprint.rb +21 -7
- data/lib/recog/fingerprint_parse_error.rb +10 -0
- data/lib/recog/match_reporter.rb +37 -3
- data/lib/recog/matcher.rb +5 -10
- data/lib/recog/verifier.rb +4 -4
- data/lib/recog/verify_reporter.rb +7 -6
- data/lib/recog/version.rb +1 -1
- data/{bin → recog/bin}/recog_match +20 -7
- data/{xml → recog/xml}/apache_modules.xml +0 -0
- data/{xml → recog/xml}/apache_os.xml +61 -19
- data/{xml → recog/xml}/architecture.xml +15 -1
- data/{xml → recog/xml}/dhcp_vendor_class.xml +10 -10
- data/{xml → recog/xml}/dns_versionbind.xml +16 -13
- data/{xml → recog/xml}/favicons.xml +167 -9
- data/{xml → recog/xml}/fingerprints.xsd +9 -1
- data/{xml → recog/xml}/ftp_banners.xml +131 -141
- data/{xml → recog/xml}/h323_callresp.xml +2 -2
- data/{xml → recog/xml}/hp_pjl_id.xml +81 -81
- data/{xml → recog/xml}/html_title.xml +250 -9
- data/{xml → recog/xml}/http_cookies.xml +111 -34
- data/{xml → recog/xml}/http_servers.xml +483 -270
- data/{xml → recog/xml}/http_wwwauth.xml +83 -37
- data/{xml → recog/xml}/imap_banners.xml +10 -10
- data/{xml → recog/xml}/ldap_searchresult.xml +0 -0
- data/{xml → recog/xml}/mdns_device-info_txt.xml +0 -0
- data/{xml → recog/xml}/mdns_workstation_txt.xml +0 -0
- data/{xml → recog/xml}/mysql_banners.xml +0 -0
- data/{xml → recog/xml}/mysql_error.xml +0 -0
- data/{xml → recog/xml}/nntp_banners.xml +8 -5
- data/{xml → recog/xml}/ntp_banners.xml +33 -33
- data/{xml → recog/xml}/operating_system.xml +92 -77
- data/{xml → recog/xml}/pop_banners.xml +25 -25
- data/{xml → recog/xml}/rsh_resp.xml +0 -0
- data/{xml → recog/xml}/rtsp_servers.xml +0 -0
- data/{xml → recog/xml}/sip_banners.xml +16 -5
- data/{xml → recog/xml}/sip_user_agents.xml +122 -27
- data/{xml → recog/xml}/smb_native_lm.xml +5 -5
- data/{xml → recog/xml}/smb_native_os.xml +25 -25
- data/{xml → recog/xml}/smtp_banners.xml +132 -131
- data/{xml → recog/xml}/smtp_debug.xml +0 -0
- data/{xml → recog/xml}/smtp_ehlo.xml +0 -0
- data/{xml → recog/xml}/smtp_expn.xml +0 -0
- data/{xml → recog/xml}/smtp_help.xml +1 -1
- data/{xml → recog/xml}/smtp_mailfrom.xml +0 -0
- data/{xml → recog/xml}/smtp_noop.xml +0 -0
- data/{xml → recog/xml}/smtp_quit.xml +0 -0
- data/{xml → recog/xml}/smtp_rcptto.xml +0 -0
- data/{xml → recog/xml}/smtp_rset.xml +0 -0
- data/{xml → recog/xml}/smtp_turn.xml +0 -0
- data/{xml → recog/xml}/smtp_vrfy.xml +0 -0
- data/{xml → recog/xml}/snmp_sysdescr.xml +1248 -1233
- data/{xml → recog/xml}/snmp_sysobjid.xml +13 -2
- data/{xml → recog/xml}/ssh_banners.xml +9 -5
- data/{xml → recog/xml}/telnet_banners.xml +83 -1
- data/{xml → recog/xml}/tls_jarm.xml +30 -2
- data/{xml → recog/xml}/x11_banners.xml +3 -3
- data/{xml → recog/xml}/x509_issuers.xml +24 -4
- data/{xml → recog/xml}/x509_subjects.xml +32 -3
- data/recog.gemspec +9 -5
- data/spec/data/external_example_fingerprint/hp_printer_ex_01.txt +1 -0
- data/spec/data/external_example_fingerprint/hp_printer_ex_02.txt +1 -0
- data/spec/data/external_example_fingerprint.xml +8 -0
- data/spec/data/external_example_illegal_path_fingerprint.xml +7 -0
- data/spec/lib/recog/db_spec.rb +84 -61
- data/spec/lib/recog/fingerprint_spec.rb +4 -4
- data/spec/lib/recog/match_reporter_spec.rb +22 -8
- data/spec/lib/recog/verify_reporter_spec.rb +8 -8
- data/spec/spec_helper.rb +4 -0
- data.tar.gz.sig +0 -0
- metadata +154 -142
- metadata.gz.sig +0 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +0 -37
- data/.github/ISSUE_TEMPLATE/feature_request.md +0 -17
- data/.github/ISSUE_TEMPLATE/fingerprint_request.md +0 -27
- data/.github/PULL_REQUEST_TEMPLATE +0 -24
- data/.github/SECURITY.md +0 -35
- data/.github/dependabot.yml +0 -8
- data/.github/workflows/ci.yml +0 -26
- data/.github/workflows/verify.yml +0 -89
- data/.gitignore +0 -23
- data/.rspec +0 -3
- data/.ruby-gemset +0 -1
- data/.ruby-version +0 -1
- data/.snyk +0 -10
- data/.travis.yml +0 -25
- data/CONTRIBUTING.md +0 -276
- data/bin/recog_cleanup +0 -16
- data/bin/recog_export +0 -81
- data/bin/recog_standardize +0 -163
- data/bin/recog_verify +0 -63
- data/cpe-remap.yaml +0 -356
- data/features/data/failing_banners_fingerprints.xml +0 -20
- data/features/data/matching_banners_fingerprints.xml +0 -23
- data/features/data/multiple_banners_fingerprints.xml +0 -32
- data/features/data/no_tests.xml +0 -3
- data/features/data/sample_banner.txt +0 -2
- data/features/data/successful_tests.xml +0 -18
- data/features/data/tests_with_failures.xml +0 -20
- data/features/data/tests_with_warnings.xml +0 -17
- data/features/match.feature +0 -36
- data/features/support/aruba.rb +0 -3
- data/features/support/env.rb +0 -6
- data/features/verify.feature +0 -48
- data/identifiers/README.md +0 -70
- data/identifiers/fields.txt +0 -105
- data/identifiers/hw_device.txt +0 -84
- data/identifiers/hw_family.txt +0 -121
- data/identifiers/hw_product.txt +0 -461
- data/identifiers/os_architecture.txt +0 -10
- data/identifiers/os_device.txt +0 -75
- data/identifiers/os_family.txt +0 -234
- data/identifiers/os_product.txt +0 -350
- data/identifiers/service_family.txt +0 -249
- data/identifiers/service_product.txt +0 -764
- data/identifiers/vendor.txt +0 -847
- data/lib/recog/verifier_factory.rb +0 -13
- data/misc/convert_mysql_err +0 -61
- data/misc/order.xsl +0 -17
- data/requirements.txt +0 -2
- data/spec/lib/fingerprint_self_test_spec.rb +0 -175
- data/tools/dev/hooks/pre-commit +0 -21
- data/update_cpes.py +0 -250
@@ -1,13 +0,0 @@
|
|
1
|
-
require 'recog/verifier'
|
2
|
-
require 'recog/formatter'
|
3
|
-
require 'recog/verify_reporter'
|
4
|
-
|
5
|
-
module Recog
|
6
|
-
module VerifierFactory
|
7
|
-
def self.build(options, db)
|
8
|
-
formatter = Formatter.new(options, $stdout)
|
9
|
-
reporter = VerifyReporter.new(options, formatter, db.path)
|
10
|
-
Verifier.new(db, reporter)
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
data/misc/convert_mysql_err
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# Takes the MySQL error messages from sql/share/errmsg-utf8.txt, locates the
|
4
|
-
# provided error message type (for example, ER_HOST_NOT_PRIVILEGED), then
|
5
|
-
# creates XML snippets for each locale to be used in Recog. Note that this
|
6
|
-
# cannot be used as-is to generate mysql_errors.xml, or oftentimes even parts
|
7
|
-
# -- it merely spits out XML snippets that you can start with; many will still
|
8
|
-
# need to be modified by hand.
|
9
|
-
|
10
|
-
require 'builder'
|
11
|
-
require 'open-uri'
|
12
|
-
require 'securerandom'
|
13
|
-
|
14
|
-
def generate_recog(error_name, locale, error_message)
|
15
|
-
xml = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
|
16
|
-
xml.fingerprint(pattern: error_message) do
|
17
|
-
xml.description "Oracle MySQL error #{error_name} (#{locale})"
|
18
|
-
xml.example(error_message)
|
19
|
-
xml.param(pos: 0, name: 'service.vendor', value: 'Oracle')
|
20
|
-
xml.param(pos: 0, name: 'service.family', value: 'MySQL')
|
21
|
-
xml.param(pos: 0, name: 'service.product', value: 'MySQL')
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
unless ARGV.size == 2
|
26
|
-
fail "Usage: #{$PROGRAM_NAME} <path/URI for errmsg-utf8.txt> <error name>"
|
27
|
-
end
|
28
|
-
|
29
|
-
path = ARGV.first
|
30
|
-
error_name = ARGV.last
|
31
|
-
|
32
|
-
lines = IO.readlines(open(path))
|
33
|
-
|
34
|
-
fail "Nothing read from #{path}" if lines.empty?
|
35
|
-
|
36
|
-
unless (error_start = lines.find_index { |line| line.strip =~ /^#{error_name}(?:\s+\S+)?$/ })
|
37
|
-
fail "Unable to find #{error_name} in #{path}"
|
38
|
-
end
|
39
|
-
|
40
|
-
locale_map = {}
|
41
|
-
lines.slice(error_start + 1, lines.size).each do |line|
|
42
|
-
if /^\s+(?<locale>\S+)\s+"(?<error_message>.*)",?$/ =~ line
|
43
|
-
locale_map[locale] = error_message
|
44
|
-
else
|
45
|
-
break
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
# Many of the error messages contain format strings. This can be problematic
|
50
|
-
# in that they need to be removed or otherwise handled as part of the 'pattern'
|
51
|
-
# attribute and appropriately filled in in any example elements. So simply try
|
52
|
-
# a rough count of the possible format strings and warn the user so that they
|
53
|
-
# can deal with it.
|
54
|
-
format_count = locale_map.values.map { |error_message| error_message.scan(/%/).size }.inject(&:+)
|
55
|
-
unless format_count == 0
|
56
|
-
warn("#{format_count} possible format strings found -- you'll need to deal with this")
|
57
|
-
end
|
58
|
-
|
59
|
-
Hash[locale_map.sort].map do |locale, error_message|
|
60
|
-
generate_recog(error_name, locale, error_message)
|
61
|
-
end
|
data/misc/order.xsl
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
<?xml version="1.0"?>
|
2
|
-
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
|
3
|
-
<xsl:output encoding="UTF-8" indent="yes" method="xml"/>
|
4
|
-
<xsl:template match="@*|node()">
|
5
|
-
<xsl:copy>
|
6
|
-
<xsl:apply-templates select="@*|node()"/>
|
7
|
-
</xsl:copy>
|
8
|
-
</xsl:template>
|
9
|
-
<xsl:template match="fingerprints/fingerprint">
|
10
|
-
<xsl:copy>
|
11
|
-
<xsl:copy-of select="@*"/>
|
12
|
-
<xsl:apply-templates select="description"/>
|
13
|
-
<xsl:apply-templates select="example"/>
|
14
|
-
<xsl:apply-templates select="param"/>
|
15
|
-
</xsl:copy>
|
16
|
-
</xsl:template>
|
17
|
-
</xsl:stylesheet>
|
data/requirements.txt
DELETED
@@ -1,175 +0,0 @@
|
|
1
|
-
require 'recog/db'
|
2
|
-
require 'regexp_parser'
|
3
|
-
require 'nokogiri'
|
4
|
-
|
5
|
-
describe Recog::DB do
|
6
|
-
let(:schema) { Nokogiri::XML::Schema(open(File.expand_path(File.join(%w(xml fingerprints.xsd))))) }
|
7
|
-
Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
|
8
|
-
|
9
|
-
describe "##{File.basename(xml_file_name)}" do
|
10
|
-
|
11
|
-
it "is valid XML" do
|
12
|
-
doc = Nokogiri::XML(open(xml_file_name))
|
13
|
-
errors = schema.validate(doc)
|
14
|
-
expect(errors).to be_empty, "#{xml_file_name} is invalid recog XML -- #{errors.inspect}"
|
15
|
-
end
|
16
|
-
|
17
|
-
db = Recog::DB.new(xml_file_name)
|
18
|
-
|
19
|
-
it "has a match key" do
|
20
|
-
expect(db.match_key).not_to be_nil
|
21
|
-
expect(db.match_key).not_to be_empty
|
22
|
-
end
|
23
|
-
|
24
|
-
it "has valid 'preference' value" do
|
25
|
-
# Reserve values below 0.10 and above 0.90 for users
|
26
|
-
# See xml/fingerprints.xsd
|
27
|
-
expect(db.preference.class).to be ::Float
|
28
|
-
expect(db.preference).to be_between(0.10, 0.90)
|
29
|
-
end
|
30
|
-
|
31
|
-
fp_descriptions = []
|
32
|
-
db.fingerprints.each_index do |i|
|
33
|
-
fp = db.fingerprints[i]
|
34
|
-
|
35
|
-
it "doesn't have a duplicate description" do
|
36
|
-
if fp_descriptions.include?(fp.name)
|
37
|
-
fail "'#{fp.name}'s description is not unique"
|
38
|
-
else
|
39
|
-
fp_descriptions << fp.name
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
context "#{fp.name}" do
|
44
|
-
param_names = []
|
45
|
-
it "has consistent os.device and hw.device" do
|
46
|
-
if fp.params['os.device'] && fp.params['hw.device'] && (fp.params['os.device'] != fp.params['hw.device'])
|
47
|
-
fail "#{fp.name} has both hw.device and os.device but with differing values"
|
48
|
-
end
|
49
|
-
end
|
50
|
-
fp.params.each do |param_name, pos_value|
|
51
|
-
pos, value = pos_value
|
52
|
-
it "has valid looking fingerprint parameter names" do
|
53
|
-
unless param_name =~ /^(?:cookie|[^\.]+\..*)$/
|
54
|
-
fail "'#{param_name}' is invalid"
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
it "doesn't have param values for capture params" do
|
59
|
-
if pos > 0 && !value.to_s.empty?
|
60
|
-
fail "'#{fp.name}'s #{param_name} is a non-zero pos but specifies a value of '#{value}'"
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
it "has parameter values other than General, Server or Unknown, which are not helpful" do
|
65
|
-
if pos == 0 && value =~ /^(?i:general|server|unknown)$/
|
66
|
-
fail "'#{param_name}' has general/server/unknown value '#{value}'"
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
it "doesn't omit values for non-capture params" do
|
71
|
-
if pos == 0 && value.to_s.empty?
|
72
|
-
fail "'#{fp.name}'s #{param_name} is not a capture (pos=0) but doesn't specify a value"
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
it "doesn't have duplicate params" do
|
77
|
-
if param_names.include?(param_name)
|
78
|
-
fail "'#{fp.name}'s has duplicate #{param_name}"
|
79
|
-
else
|
80
|
-
param_names << param_name
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
it "uses interpolation correctly" do
|
85
|
-
if pos == 0 && /\{(?<interpolated>[^\s{}]+)\}/ =~ value
|
86
|
-
unless fp.params.key?(interpolated)
|
87
|
-
fail "'#{fp.name}' uses interpolated value '#{interpolated}' that does not exist"
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
context "#{fp.regex}" do
|
95
|
-
|
96
|
-
it "has a valid looking name" do
|
97
|
-
expect(fp.name).not_to be_nil
|
98
|
-
expect(fp.name).not_to be_empty
|
99
|
-
end
|
100
|
-
|
101
|
-
it "has a regex" do
|
102
|
-
expect(fp.regex).not_to be_nil
|
103
|
-
expect(fp.regex.class).to be ::Regexp
|
104
|
-
end
|
105
|
-
|
106
|
-
it 'uses capturing regular expressions properly' do
|
107
|
-
# the list of index-based captures that the fingerprint is expecting
|
108
|
-
expected_capture_positions = fp.params.values.map(&:first).map(&:to_i).select { |position| position > 0 }
|
109
|
-
if fp.params.empty? && expected_capture_positions.size > 0
|
110
|
-
fail "Non-asserting fingerprint with regex #{fp.regex} captures #{expected_capture_positions.size} time(s); 0 are needed"
|
111
|
-
else
|
112
|
-
# parse the regex and count the number of captures
|
113
|
-
actual_capture_positions = []
|
114
|
-
capture_number = 1
|
115
|
-
Regexp::Scanner.scan(fp.regex).each do |token_parts|
|
116
|
-
if token_parts.first == :group && ![:close, :passive, :options, :options_switch].include?(token_parts[1])
|
117
|
-
actual_capture_positions << capture_number
|
118
|
-
capture_number += 1
|
119
|
-
end
|
120
|
-
end
|
121
|
-
# compare the captures actually performed to those being used and ensure that they contain
|
122
|
-
# the same elements regardless of order, preventing, over-, under- and other forms of mis-capturing.
|
123
|
-
actual_capture_positions = actual_capture_positions.sort.uniq
|
124
|
-
expected_capture_positions = expected_capture_positions.sort.uniq
|
125
|
-
expect(actual_capture_positions).to eq(expected_capture_positions),
|
126
|
-
"Regex has #{actual_capture_positions.size} capture groups, but the fingerprint expected #{expected_capture_positions.size} extractions."
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
# Not yet enforced
|
131
|
-
# it "has test cases" do
|
132
|
-
# expect(fp.tests.length).not_to equal(0)
|
133
|
-
# end
|
134
|
-
|
135
|
-
it "Has a reasonable number (<= 20) of test cases" do
|
136
|
-
expect(fp.tests.length).to be <= 20
|
137
|
-
end
|
138
|
-
|
139
|
-
fp_examples = []
|
140
|
-
fp.tests.each do |example|
|
141
|
-
it "doesn't have a duplicate examples" do
|
142
|
-
if fp_examples.include?(example.content)
|
143
|
-
fail "'#{fp.name}' has duplicate example '#{example.content}'"
|
144
|
-
else
|
145
|
-
fp_examples << example.content
|
146
|
-
end
|
147
|
-
end
|
148
|
-
it "Example '#{example.content}' matches this regex" do
|
149
|
-
match = fp.match(example.content)
|
150
|
-
expect(match).to_not be_nil, 'Regex did not match'
|
151
|
-
# test any extractions specified in the example
|
152
|
-
example.attributes.each_pair do |k,v|
|
153
|
-
next if k == '_encoding'
|
154
|
-
next if k == '_filename'
|
155
|
-
expect(match[k]).to eq(v), "Regex didn't extract expected value for fingerprint attribute #{k} -- got #{match[k]} instead of #{v}"
|
156
|
-
end
|
157
|
-
end
|
158
|
-
|
159
|
-
it "Example '#{example.content}' matches this regex first" do
|
160
|
-
db.fingerprints.slice(0, i).each_index do |previous_i|
|
161
|
-
prev_fp = db.fingerprints[previous_i]
|
162
|
-
prev_fp.tests.each do |prev_example|
|
163
|
-
match = prev_fp.match(example.content)
|
164
|
-
expect(match).to be_nil, "Matched regex ##{previous_i} (#{db.fingerprints[previous_i].regex}) rather than ##{i} (#{db.fingerprints[i].regex})"
|
165
|
-
end
|
166
|
-
end
|
167
|
-
end
|
168
|
-
end
|
169
|
-
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
data/tools/dev/hooks/pre-commit
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
#!/bin/sh
|
2
|
-
#
|
3
|
-
# Hook script to verify changes about to be committed.
|
4
|
-
# The hook should exit with non-zero status after issuing an appropriate
|
5
|
-
# message if it wants to stop the commit.
|
6
|
-
|
7
|
-
# Verify that each fingerprint asserts known identifiers.
|
8
|
-
git diff --cached --name-only --diff-filter=ACM -z xml/*.xml | xargs -0 ./bin/recog_standardize --write
|
9
|
-
|
10
|
-
# get status
|
11
|
-
status=$?
|
12
|
-
|
13
|
-
if [ $status -ne 0 ]; then
|
14
|
-
echo "Please review any new additions to the text files under 'identifiers/'."
|
15
|
-
echo "If any of these names are close to an existing name, update the offending"
|
16
|
-
echo "fingerprint to use the existing name instead. Once the fingerprints are fixed,"
|
17
|
-
echo "remove the 'extra' names from the identifiers files, and run the tool again."
|
18
|
-
exit 1
|
19
|
-
fi
|
20
|
-
|
21
|
-
exit 0
|
data/update_cpes.py
DELETED
@@ -1,250 +0,0 @@
|
|
1
|
-
#!/usr/bin/env python
|
2
|
-
|
3
|
-
import logging
|
4
|
-
import re
|
5
|
-
import sys
|
6
|
-
|
7
|
-
import yaml
|
8
|
-
from lxml import etree
|
9
|
-
|
10
|
-
def parse_r7_remapping(file):
|
11
|
-
with open(file) as remap_file:
|
12
|
-
return yaml.safe_load(remap_file)["mappings"]
|
13
|
-
|
14
|
-
def parse_cpe_vp_map(file):
|
15
|
-
vp_map = {} # cpe_type -> vendor -> products
|
16
|
-
parser = etree.XMLParser(remove_comments=False)
|
17
|
-
doc = etree.parse(file, parser)
|
18
|
-
namespaces = {'ns': 'http://cpe.mitre.org/dictionary/2.0', 'meta': 'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2'}
|
19
|
-
for entry in doc.xpath("//ns:cpe-list/ns:cpe-item", namespaces=namespaces):
|
20
|
-
cpe_name = entry.get("name")
|
21
|
-
if not cpe_name:
|
22
|
-
continue
|
23
|
-
|
24
|
-
# If the entry is deprecated then don't add it to our list of valid CPEs.
|
25
|
-
if entry.get("deprecated"):
|
26
|
-
continue
|
27
|
-
|
28
|
-
cpe_match = re.match('^cpe:/([aho]):([^:]+):([^:]+)', cpe_name)
|
29
|
-
|
30
|
-
if cpe_match:
|
31
|
-
cpe_type, vendor, product = cpe_match.group(1, 2, 3)
|
32
|
-
if cpe_type not in vp_map:
|
33
|
-
vp_map[cpe_type] = {}
|
34
|
-
if vendor not in vp_map[cpe_type]:
|
35
|
-
vp_map[cpe_type][vendor] = set()
|
36
|
-
product = product.replace('%2f', '/')
|
37
|
-
vp_map[cpe_type][vendor].add(product)
|
38
|
-
else:
|
39
|
-
logging.error("Unexpected CPE %s", cpe_name)
|
40
|
-
|
41
|
-
return vp_map
|
42
|
-
|
43
|
-
def main():
|
44
|
-
if len(sys.argv) != 4:
|
45
|
-
logging.critical("Expecting exactly 3 arguments; recog XML file, CPE 2.3 XML dictionary, JSON remapping, got %s", (len(sys.argv) - 1))
|
46
|
-
sys.exit(1)
|
47
|
-
|
48
|
-
cpe_vp_map = parse_cpe_vp_map(sys.argv[2])
|
49
|
-
if not cpe_vp_map:
|
50
|
-
logging.critical("No CPE vendor => product mappings read from CPE 2.3 XML dictionary %s", sys.argv[2])
|
51
|
-
sys.exit(1)
|
52
|
-
|
53
|
-
r7_vp_map = parse_r7_remapping(sys.argv[3])
|
54
|
-
if not r7_vp_map:
|
55
|
-
logging.warning("No Rapid7 vendor/product => CPE mapping read from %s", sys.argv[3])
|
56
|
-
|
57
|
-
update_cpes(sys.argv[1], cpe_vp_map, r7_vp_map)
|
58
|
-
|
59
|
-
def lookup_cpe(vendor, product, cpe_type, cpe_table, remap):
|
60
|
-
"""Identify the correct vendor and product values for a CPE
|
61
|
-
|
62
|
-
This function attempts to determine the correct CPE using vendor and product
|
63
|
-
values supplied by the caller as well as a remapping dictionary for mapping
|
64
|
-
these values to more correct values used by NIST.
|
65
|
-
|
66
|
-
For example, the remapping might tell us that a value of 'alpine' for the
|
67
|
-
vendor string should be 'alpinelinux' instead, or for product 'solaris'
|
68
|
-
should be 'sunos'.
|
69
|
-
|
70
|
-
This function should only emit values seen in the official NIST CPE list
|
71
|
-
which is provided to it in cpe_table.
|
72
|
-
|
73
|
-
Lookup priority:
|
74
|
-
1. Original vendor / product
|
75
|
-
2. Original vendor / remap product
|
76
|
-
3. Remap vendor / original product
|
77
|
-
4. Remap vendor / remap product
|
78
|
-
|
79
|
-
Args:
|
80
|
-
vendor (str): vendor name
|
81
|
-
product (str): product name
|
82
|
-
cpe_type (str): CPE type - o, a, h, etc.
|
83
|
-
cpe_table (dict): dict containing the official NIST CPE data
|
84
|
-
remap (dict): dict containing the remapping values
|
85
|
-
Returns:
|
86
|
-
success, vendor, product
|
87
|
-
"""
|
88
|
-
|
89
|
-
if (
|
90
|
-
vendor in cpe_table[cpe_type]
|
91
|
-
and product in cpe_table[cpe_type][vendor]
|
92
|
-
):
|
93
|
-
# Hot path, success with original values
|
94
|
-
return True, vendor, product
|
95
|
-
|
96
|
-
# Everything else depends on a remap of some sort.
|
97
|
-
# get the remappings for this one vendor string.
|
98
|
-
vendor_remap = None
|
99
|
-
|
100
|
-
remap_type = remap.get(cpe_type, None)
|
101
|
-
if remap_type:
|
102
|
-
vendor_remap = remap_type.get(vendor, None)
|
103
|
-
|
104
|
-
if vendor_remap:
|
105
|
-
# If we have product remappings, work that angle next
|
106
|
-
possible_product = None
|
107
|
-
if (
|
108
|
-
vendor_remap.get('products', None)
|
109
|
-
and product in vendor_remap['products']
|
110
|
-
):
|
111
|
-
possible_product = vendor_remap['products'][product]
|
112
|
-
|
113
|
-
if (vendor in cpe_table[cpe_type]
|
114
|
-
and possible_product
|
115
|
-
and possible_product in cpe_table[cpe_type][vendor]):
|
116
|
-
# Found original vendor, remap product
|
117
|
-
return True, vendor, possible_product
|
118
|
-
|
119
|
-
# Start working the process to find a match with a remapped vendor name
|
120
|
-
if vendor_remap.get('vendor', None):
|
121
|
-
new_vendor = vendor_remap['vendor']
|
122
|
-
|
123
|
-
if new_vendor in cpe_table[cpe_type]:
|
124
|
-
|
125
|
-
if product in cpe_table[cpe_type][new_vendor]:
|
126
|
-
# Found remap vendor, original product
|
127
|
-
return True, new_vendor, product
|
128
|
-
|
129
|
-
if possible_product and possible_product in cpe_table[cpe_type][new_vendor]:
|
130
|
-
# Found remap vendor, remap product
|
131
|
-
return True, new_vendor, possible_product
|
132
|
-
|
133
|
-
|
134
|
-
logging.error("Product %s from vendor %s invalid for CPE %s and no mapping",
|
135
|
-
product, vendor, cpe_type)
|
136
|
-
return False, None, None
|
137
|
-
|
138
|
-
|
139
|
-
def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
|
140
|
-
parser = etree.XMLParser(remove_comments=False, remove_blank_text=True)
|
141
|
-
doc = etree.parse(xml_file, parser)
|
142
|
-
|
143
|
-
for fingerprint in doc.xpath('//fingerprint'):
|
144
|
-
|
145
|
-
# collect all the params, grouping by os and service params that could be used to compute a CPE
|
146
|
-
params = {}
|
147
|
-
for param in fingerprint.xpath('./param'):
|
148
|
-
name = param.attrib['name']
|
149
|
-
# remove any existing CPE params
|
150
|
-
if re.match(r'^.*\.cpe\d{0,2}$', name):
|
151
|
-
param.getparent().remove(param)
|
152
|
-
continue
|
153
|
-
|
154
|
-
match = re.search(r'^(?P<fp_type>hw|os|service(?:\.component)?)\.', name)
|
155
|
-
if match:
|
156
|
-
fp_type = match.group('fp_type')
|
157
|
-
if not fp_type in params:
|
158
|
-
params[fp_type] = {}
|
159
|
-
if name in params[fp_type]:
|
160
|
-
raise ValueError('Duplicated fingerprint named {} in fingerprint {} in file {}'.format(name, fingerprint.attrib['pattern'], xml_file))
|
161
|
-
params[fp_type][name] = param
|
162
|
-
|
163
|
-
|
164
|
-
# for each of the applicable os/service param groups, build a CPE
|
165
|
-
for fp_type in params:
|
166
|
-
if fp_type == 'os':
|
167
|
-
cpe_type = 'o'
|
168
|
-
elif fp_type.startswith('service'):
|
169
|
-
cpe_type = 'a'
|
170
|
-
elif fp_type == 'hw':
|
171
|
-
cpe_type = 'h'
|
172
|
-
else:
|
173
|
-
raise ValueError('Unhandled param type {}'.format(fp_type))
|
174
|
-
|
175
|
-
# extract the vendor/product/version values from each os/service group,
|
176
|
-
# using the static value ('Apache', for example) when pos is 0, and
|
177
|
-
# otherwise use a value that contains interpolation markers such that
|
178
|
-
# products/projects that use recog content can insert the value
|
179
|
-
# extracted from the banner/other data via regex capturing groups
|
180
|
-
fp_data = {
|
181
|
-
'vendor': None,
|
182
|
-
'product': None,
|
183
|
-
'version': '-',
|
184
|
-
}
|
185
|
-
for fp_datum in fp_data:
|
186
|
-
fp_datum_param_name = "{}.{}".format(fp_type, fp_datum)
|
187
|
-
if fp_datum_param_name in params[fp_type]:
|
188
|
-
fp_datum_e = params[fp_type][fp_datum_param_name]
|
189
|
-
if fp_datum_e.attrib['pos'] == '0':
|
190
|
-
fp_data[fp_datum] = fp_datum_e.attrib['value']
|
191
|
-
else:
|
192
|
-
fp_data[fp_datum] = "{{{}}}".format(fp_datum_e.attrib['name'])
|
193
|
-
|
194
|
-
vendor = fp_data['vendor']
|
195
|
-
product = fp_data['product']
|
196
|
-
version = fp_data['version']
|
197
|
-
|
198
|
-
# build a reasonable looking CPE value from the vendor/product/version,
|
199
|
-
# lowercasing, replacing whitespace with _, and more
|
200
|
-
if vendor and product:
|
201
|
-
if not cpe_type in cpe_vp_map:
|
202
|
-
logging.error("Didn't find CPE type '%s' for '%s' '%s'", cpe_type, vendor, product)
|
203
|
-
continue
|
204
|
-
|
205
|
-
vendor = vendor.lower().replace(' ', '_').replace(',', '')
|
206
|
-
product = product.lower().replace(' ', '_').replace(',', '').replace('!', '%21')
|
207
|
-
if 'unknown' in [vendor, product]:
|
208
|
-
continue
|
209
|
-
|
210
|
-
if (vendor.startswith('{') and vendor.endswith('}')) or (product.startswith('{') and product.endswith('}')):
|
211
|
-
continue
|
212
|
-
|
213
|
-
success, vendor, product = lookup_cpe(vendor, product, cpe_type, cpe_vp_map, r7_vp_map)
|
214
|
-
if not success:
|
215
|
-
continue
|
216
|
-
|
217
|
-
# Sanity check the value to ensure that no invalid values will
|
218
|
-
# slip in due to logic or mapping bugs.
|
219
|
-
# If it's not in the official NIST list then log it and kick it out
|
220
|
-
if product not in cpe_vp_map[cpe_type][vendor]:
|
221
|
-
logging.error("Invalid CPE type %s created for vendor %s and product %s. This may be due to an invalid mapping.", cpe_type, vendor, product)
|
222
|
-
continue
|
223
|
-
|
224
|
-
# building the CPE string
|
225
|
-
# Last minute escaping of '/' and `!`
|
226
|
-
product = product.replace('/', '\/').replace('%21', '\!')
|
227
|
-
cpe_value = 'cpe:/{}:{}:{}'.format(cpe_type, vendor, product)
|
228
|
-
|
229
|
-
if version:
|
230
|
-
cpe_value += ":{}".format(version)
|
231
|
-
|
232
|
-
cpe_param = etree.Element('param')
|
233
|
-
cpe_param.attrib['pos'] = '0'
|
234
|
-
cpe_param.attrib['name'] = '{}.cpe23'.format(fp_type)
|
235
|
-
cpe_param.attrib['value'] = cpe_value
|
236
|
-
|
237
|
-
for param_name in params[fp_type]:
|
238
|
-
param = params[fp_type][param_name]
|
239
|
-
parent = param.getparent()
|
240
|
-
index = parent.index(param) + 1
|
241
|
-
parent.insert(index, cpe_param)
|
242
|
-
|
243
|
-
root = doc.getroot()
|
244
|
-
|
245
|
-
with open(xml_file, 'wb') as xml_out:
|
246
|
-
xml_out.write(etree.tostring(root, pretty_print=True, xml_declaration=True, encoding=doc.docinfo.encoding))
|
247
|
-
|
248
|
-
if __name__ == '__main__':
|
249
|
-
try: sys.exit(main())
|
250
|
-
except KeyboardInterrupt: pass
|