recog 2.3.22 → 3.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +2 -0
  3. data/LICENSE +1 -1
  4. data/README.md +25 -16
  5. data/Rakefile +2 -9
  6. data/lib/recog/db_manager.rb +1 -1
  7. data/lib/recog/fingerprint.rb +21 -7
  8. data/lib/recog/fingerprint_parse_error.rb +10 -0
  9. data/lib/recog/match_reporter.rb +37 -3
  10. data/lib/recog/matcher.rb +5 -10
  11. data/lib/recog/verifier.rb +4 -4
  12. data/lib/recog/verify_reporter.rb +7 -6
  13. data/lib/recog/version.rb +1 -1
  14. data/{bin → recog/bin}/recog_match +20 -7
  15. data/{xml → recog/xml}/apache_modules.xml +0 -0
  16. data/{xml → recog/xml}/apache_os.xml +61 -19
  17. data/{xml → recog/xml}/architecture.xml +15 -1
  18. data/{xml → recog/xml}/dhcp_vendor_class.xml +10 -10
  19. data/{xml → recog/xml}/dns_versionbind.xml +16 -13
  20. data/{xml → recog/xml}/favicons.xml +167 -9
  21. data/{xml → recog/xml}/fingerprints.xsd +9 -1
  22. data/{xml → recog/xml}/ftp_banners.xml +131 -141
  23. data/{xml → recog/xml}/h323_callresp.xml +2 -2
  24. data/{xml → recog/xml}/hp_pjl_id.xml +81 -81
  25. data/{xml → recog/xml}/html_title.xml +250 -9
  26. data/{xml → recog/xml}/http_cookies.xml +111 -34
  27. data/{xml → recog/xml}/http_servers.xml +483 -270
  28. data/{xml → recog/xml}/http_wwwauth.xml +83 -37
  29. data/{xml → recog/xml}/imap_banners.xml +10 -10
  30. data/{xml → recog/xml}/ldap_searchresult.xml +0 -0
  31. data/{xml → recog/xml}/mdns_device-info_txt.xml +0 -0
  32. data/{xml → recog/xml}/mdns_workstation_txt.xml +0 -0
  33. data/{xml → recog/xml}/mysql_banners.xml +0 -0
  34. data/{xml → recog/xml}/mysql_error.xml +0 -0
  35. data/{xml → recog/xml}/nntp_banners.xml +8 -5
  36. data/{xml → recog/xml}/ntp_banners.xml +33 -33
  37. data/{xml → recog/xml}/operating_system.xml +92 -77
  38. data/{xml → recog/xml}/pop_banners.xml +25 -25
  39. data/{xml → recog/xml}/rsh_resp.xml +0 -0
  40. data/{xml → recog/xml}/rtsp_servers.xml +0 -0
  41. data/{xml → recog/xml}/sip_banners.xml +16 -5
  42. data/{xml → recog/xml}/sip_user_agents.xml +122 -27
  43. data/{xml → recog/xml}/smb_native_lm.xml +5 -5
  44. data/{xml → recog/xml}/smb_native_os.xml +25 -25
  45. data/{xml → recog/xml}/smtp_banners.xml +132 -131
  46. data/{xml → recog/xml}/smtp_debug.xml +0 -0
  47. data/{xml → recog/xml}/smtp_ehlo.xml +0 -0
  48. data/{xml → recog/xml}/smtp_expn.xml +0 -0
  49. data/{xml → recog/xml}/smtp_help.xml +1 -1
  50. data/{xml → recog/xml}/smtp_mailfrom.xml +0 -0
  51. data/{xml → recog/xml}/smtp_noop.xml +0 -0
  52. data/{xml → recog/xml}/smtp_quit.xml +0 -0
  53. data/{xml → recog/xml}/smtp_rcptto.xml +0 -0
  54. data/{xml → recog/xml}/smtp_rset.xml +0 -0
  55. data/{xml → recog/xml}/smtp_turn.xml +0 -0
  56. data/{xml → recog/xml}/smtp_vrfy.xml +0 -0
  57. data/{xml → recog/xml}/snmp_sysdescr.xml +1248 -1233
  58. data/{xml → recog/xml}/snmp_sysobjid.xml +13 -2
  59. data/{xml → recog/xml}/ssh_banners.xml +9 -5
  60. data/{xml → recog/xml}/telnet_banners.xml +83 -1
  61. data/{xml → recog/xml}/tls_jarm.xml +30 -2
  62. data/{xml → recog/xml}/x11_banners.xml +3 -3
  63. data/{xml → recog/xml}/x509_issuers.xml +24 -4
  64. data/{xml → recog/xml}/x509_subjects.xml +32 -3
  65. data/recog.gemspec +9 -5
  66. data/spec/data/external_example_fingerprint/hp_printer_ex_01.txt +1 -0
  67. data/spec/data/external_example_fingerprint/hp_printer_ex_02.txt +1 -0
  68. data/spec/data/external_example_fingerprint.xml +8 -0
  69. data/spec/data/external_example_illegal_path_fingerprint.xml +7 -0
  70. data/spec/lib/recog/db_spec.rb +84 -61
  71. data/spec/lib/recog/fingerprint_spec.rb +4 -4
  72. data/spec/lib/recog/match_reporter_spec.rb +22 -8
  73. data/spec/lib/recog/verify_reporter_spec.rb +8 -8
  74. data/spec/spec_helper.rb +4 -0
  75. data.tar.gz.sig +0 -0
  76. metadata +154 -142
  77. metadata.gz.sig +0 -0
  78. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -37
  79. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -17
  80. data/.github/ISSUE_TEMPLATE/fingerprint_request.md +0 -27
  81. data/.github/PULL_REQUEST_TEMPLATE +0 -24
  82. data/.github/SECURITY.md +0 -35
  83. data/.github/dependabot.yml +0 -8
  84. data/.github/workflows/ci.yml +0 -26
  85. data/.github/workflows/verify.yml +0 -89
  86. data/.gitignore +0 -23
  87. data/.rspec +0 -3
  88. data/.ruby-gemset +0 -1
  89. data/.ruby-version +0 -1
  90. data/.snyk +0 -10
  91. data/.travis.yml +0 -25
  92. data/CONTRIBUTING.md +0 -276
  93. data/bin/recog_cleanup +0 -16
  94. data/bin/recog_export +0 -81
  95. data/bin/recog_standardize +0 -163
  96. data/bin/recog_verify +0 -63
  97. data/cpe-remap.yaml +0 -356
  98. data/features/data/failing_banners_fingerprints.xml +0 -20
  99. data/features/data/matching_banners_fingerprints.xml +0 -23
  100. data/features/data/multiple_banners_fingerprints.xml +0 -32
  101. data/features/data/no_tests.xml +0 -3
  102. data/features/data/sample_banner.txt +0 -2
  103. data/features/data/successful_tests.xml +0 -18
  104. data/features/data/tests_with_failures.xml +0 -20
  105. data/features/data/tests_with_warnings.xml +0 -17
  106. data/features/match.feature +0 -36
  107. data/features/support/aruba.rb +0 -3
  108. data/features/support/env.rb +0 -6
  109. data/features/verify.feature +0 -48
  110. data/identifiers/README.md +0 -70
  111. data/identifiers/fields.txt +0 -105
  112. data/identifiers/hw_device.txt +0 -84
  113. data/identifiers/hw_family.txt +0 -121
  114. data/identifiers/hw_product.txt +0 -461
  115. data/identifiers/os_architecture.txt +0 -10
  116. data/identifiers/os_device.txt +0 -75
  117. data/identifiers/os_family.txt +0 -234
  118. data/identifiers/os_product.txt +0 -350
  119. data/identifiers/service_family.txt +0 -249
  120. data/identifiers/service_product.txt +0 -764
  121. data/identifiers/vendor.txt +0 -847
  122. data/lib/recog/verifier_factory.rb +0 -13
  123. data/misc/convert_mysql_err +0 -61
  124. data/misc/order.xsl +0 -17
  125. data/requirements.txt +0 -2
  126. data/spec/lib/fingerprint_self_test_spec.rb +0 -175
  127. data/tools/dev/hooks/pre-commit +0 -21
  128. data/update_cpes.py +0 -250
@@ -1,13 +0,0 @@
1
- require 'recog/verifier'
2
- require 'recog/formatter'
3
- require 'recog/verify_reporter'
4
-
5
- module Recog
6
- module VerifierFactory
7
- def self.build(options, db)
8
- formatter = Formatter.new(options, $stdout)
9
- reporter = VerifyReporter.new(options, formatter, db.path)
10
- Verifier.new(db, reporter)
11
- end
12
- end
13
- end
@@ -1,61 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- # Takes the MySQL error messages from sql/share/errmsg-utf8.txt, locates the
4
- # provided error message type (for example, ER_HOST_NOT_PRIVILEGED), then
5
- # creates XML snippets for each locale to be used in Recog. Note that this
6
- # cannot be used as-is to generate mysql_errors.xml, or oftentimes even parts
7
- # -- it merely spits out XML snippets that you can start with; many will still
8
- # need to be modified by hand.
9
-
10
- require 'builder'
11
- require 'open-uri'
12
- require 'securerandom'
13
-
14
- def generate_recog(error_name, locale, error_message)
15
- xml = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
16
- xml.fingerprint(pattern: error_message) do
17
- xml.description "Oracle MySQL error #{error_name} (#{locale})"
18
- xml.example(error_message)
19
- xml.param(pos: 0, name: 'service.vendor', value: 'Oracle')
20
- xml.param(pos: 0, name: 'service.family', value: 'MySQL')
21
- xml.param(pos: 0, name: 'service.product', value: 'MySQL')
22
- end
23
- end
24
-
25
- unless ARGV.size == 2
26
- fail "Usage: #{$PROGRAM_NAME} <path/URI for errmsg-utf8.txt> <error name>"
27
- end
28
-
29
- path = ARGV.first
30
- error_name = ARGV.last
31
-
32
- lines = IO.readlines(open(path))
33
-
34
- fail "Nothing read from #{path}" if lines.empty?
35
-
36
- unless (error_start = lines.find_index { |line| line.strip =~ /^#{error_name}(?:\s+\S+)?$/ })
37
- fail "Unable to find #{error_name} in #{path}"
38
- end
39
-
40
- locale_map = {}
41
- lines.slice(error_start + 1, lines.size).each do |line|
42
- if /^\s+(?<locale>\S+)\s+"(?<error_message>.*)",?$/ =~ line
43
- locale_map[locale] = error_message
44
- else
45
- break
46
- end
47
- end
48
-
49
- # Many of the error messages contain format strings. This can be problematic
50
- # in that they need to be removed or otherwise handled as part of the 'pattern'
51
- # attribute and appropriately filled in in any example elements. So simply try
52
- # a rough count of the possible format strings and warn the user so that they
53
- # can deal with it.
54
- format_count = locale_map.values.map { |error_message| error_message.scan(/%/).size }.inject(&:+)
55
- unless format_count == 0
56
- warn("#{format_count} possible format strings found -- you'll need to deal with this")
57
- end
58
-
59
- Hash[locale_map.sort].map do |locale, error_message|
60
- generate_recog(error_name, locale, error_message)
61
- end
data/misc/order.xsl DELETED
@@ -1,17 +0,0 @@
1
- <?xml version="1.0"?>
2
- <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
3
- <xsl:output encoding="UTF-8" indent="yes" method="xml"/>
4
- <xsl:template match="@*|node()">
5
- <xsl:copy>
6
- <xsl:apply-templates select="@*|node()"/>
7
- </xsl:copy>
8
- </xsl:template>
9
- <xsl:template match="fingerprints/fingerprint">
10
- <xsl:copy>
11
- <xsl:copy-of select="@*"/>
12
- <xsl:apply-templates select="description"/>
13
- <xsl:apply-templates select="example"/>
14
- <xsl:apply-templates select="param"/>
15
- </xsl:copy>
16
- </xsl:template>
17
- </xsl:stylesheet>
data/requirements.txt DELETED
@@ -1,2 +0,0 @@
1
- lxml==4.6.3
2
- pyyaml
@@ -1,175 +0,0 @@
1
- require 'recog/db'
2
- require 'regexp_parser'
3
- require 'nokogiri'
4
-
5
- describe Recog::DB do
6
- let(:schema) { Nokogiri::XML::Schema(open(File.expand_path(File.join(%w(xml fingerprints.xsd))))) }
7
- Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
8
-
9
- describe "##{File.basename(xml_file_name)}" do
10
-
11
- it "is valid XML" do
12
- doc = Nokogiri::XML(open(xml_file_name))
13
- errors = schema.validate(doc)
14
- expect(errors).to be_empty, "#{xml_file_name} is invalid recog XML -- #{errors.inspect}"
15
- end
16
-
17
- db = Recog::DB.new(xml_file_name)
18
-
19
- it "has a match key" do
20
- expect(db.match_key).not_to be_nil
21
- expect(db.match_key).not_to be_empty
22
- end
23
-
24
- it "has valid 'preference' value" do
25
- # Reserve values below 0.10 and above 0.90 for users
26
- # See xml/fingerprints.xsd
27
- expect(db.preference.class).to be ::Float
28
- expect(db.preference).to be_between(0.10, 0.90)
29
- end
30
-
31
- fp_descriptions = []
32
- db.fingerprints.each_index do |i|
33
- fp = db.fingerprints[i]
34
-
35
- it "doesn't have a duplicate description" do
36
- if fp_descriptions.include?(fp.name)
37
- fail "'#{fp.name}'s description is not unique"
38
- else
39
- fp_descriptions << fp.name
40
- end
41
- end
42
-
43
- context "#{fp.name}" do
44
- param_names = []
45
- it "has consistent os.device and hw.device" do
46
- if fp.params['os.device'] && fp.params['hw.device'] && (fp.params['os.device'] != fp.params['hw.device'])
47
- fail "#{fp.name} has both hw.device and os.device but with differing values"
48
- end
49
- end
50
- fp.params.each do |param_name, pos_value|
51
- pos, value = pos_value
52
- it "has valid looking fingerprint parameter names" do
53
- unless param_name =~ /^(?:cookie|[^\.]+\..*)$/
54
- fail "'#{param_name}' is invalid"
55
- end
56
- end
57
-
58
- it "doesn't have param values for capture params" do
59
- if pos > 0 && !value.to_s.empty?
60
- fail "'#{fp.name}'s #{param_name} is a non-zero pos but specifies a value of '#{value}'"
61
- end
62
- end
63
-
64
- it "has parameter values other than General, Server or Unknown, which are not helpful" do
65
- if pos == 0 && value =~ /^(?i:general|server|unknown)$/
66
- fail "'#{param_name}' has general/server/unknown value '#{value}'"
67
- end
68
- end
69
-
70
- it "doesn't omit values for non-capture params" do
71
- if pos == 0 && value.to_s.empty?
72
- fail "'#{fp.name}'s #{param_name} is not a capture (pos=0) but doesn't specify a value"
73
- end
74
- end
75
-
76
- it "doesn't have duplicate params" do
77
- if param_names.include?(param_name)
78
- fail "'#{fp.name}'s has duplicate #{param_name}"
79
- else
80
- param_names << param_name
81
- end
82
- end
83
-
84
- it "uses interpolation correctly" do
85
- if pos == 0 && /\{(?<interpolated>[^\s{}]+)\}/ =~ value
86
- unless fp.params.key?(interpolated)
87
- fail "'#{fp.name}' uses interpolated value '#{interpolated}' that does not exist"
88
- end
89
- end
90
- end
91
- end
92
- end
93
-
94
- context "#{fp.regex}" do
95
-
96
- it "has a valid looking name" do
97
- expect(fp.name).not_to be_nil
98
- expect(fp.name).not_to be_empty
99
- end
100
-
101
- it "has a regex" do
102
- expect(fp.regex).not_to be_nil
103
- expect(fp.regex.class).to be ::Regexp
104
- end
105
-
106
- it 'uses capturing regular expressions properly' do
107
- # the list of index-based captures that the fingerprint is expecting
108
- expected_capture_positions = fp.params.values.map(&:first).map(&:to_i).select { |position| position > 0 }
109
- if fp.params.empty? && expected_capture_positions.size > 0
110
- fail "Non-asserting fingerprint with regex #{fp.regex} captures #{expected_capture_positions.size} time(s); 0 are needed"
111
- else
112
- # parse the regex and count the number of captures
113
- actual_capture_positions = []
114
- capture_number = 1
115
- Regexp::Scanner.scan(fp.regex).each do |token_parts|
116
- if token_parts.first == :group && ![:close, :passive, :options, :options_switch].include?(token_parts[1])
117
- actual_capture_positions << capture_number
118
- capture_number += 1
119
- end
120
- end
121
- # compare the captures actually performed to those being used and ensure that they contain
122
- # the same elements regardless of order, preventing, over-, under- and other forms of mis-capturing.
123
- actual_capture_positions = actual_capture_positions.sort.uniq
124
- expected_capture_positions = expected_capture_positions.sort.uniq
125
- expect(actual_capture_positions).to eq(expected_capture_positions),
126
- "Regex has #{actual_capture_positions.size} capture groups, but the fingerprint expected #{expected_capture_positions.size} extractions."
127
- end
128
- end
129
-
130
- # Not yet enforced
131
- # it "has test cases" do
132
- # expect(fp.tests.length).not_to equal(0)
133
- # end
134
-
135
- it "Has a reasonable number (<= 20) of test cases" do
136
- expect(fp.tests.length).to be <= 20
137
- end
138
-
139
- fp_examples = []
140
- fp.tests.each do |example|
141
- it "doesn't have a duplicate examples" do
142
- if fp_examples.include?(example.content)
143
- fail "'#{fp.name}' has duplicate example '#{example.content}'"
144
- else
145
- fp_examples << example.content
146
- end
147
- end
148
- it "Example '#{example.content}' matches this regex" do
149
- match = fp.match(example.content)
150
- expect(match).to_not be_nil, 'Regex did not match'
151
- # test any extractions specified in the example
152
- example.attributes.each_pair do |k,v|
153
- next if k == '_encoding'
154
- next if k == '_filename'
155
- expect(match[k]).to eq(v), "Regex didn't extract expected value for fingerprint attribute #{k} -- got #{match[k]} instead of #{v}"
156
- end
157
- end
158
-
159
- it "Example '#{example.content}' matches this regex first" do
160
- db.fingerprints.slice(0, i).each_index do |previous_i|
161
- prev_fp = db.fingerprints[previous_i]
162
- prev_fp.tests.each do |prev_example|
163
- match = prev_fp.match(example.content)
164
- expect(match).to be_nil, "Matched regex ##{previous_i} (#{db.fingerprints[previous_i].regex}) rather than ##{i} (#{db.fingerprints[i].regex})"
165
- end
166
- end
167
- end
168
- end
169
-
170
- end
171
- end
172
-
173
- end
174
- end
175
- end
@@ -1,21 +0,0 @@
1
- #!/bin/sh
2
- #
3
- # Hook script to verify changes about to be committed.
4
- # The hook should exit with non-zero status after issuing an appropriate
5
- # message if it wants to stop the commit.
6
-
7
- # Verify that each fingerprint asserts known identifiers.
8
- git diff --cached --name-only --diff-filter=ACM -z xml/*.xml | xargs -0 ./bin/recog_standardize --write
9
-
10
- # get status
11
- status=$?
12
-
13
- if [ $status -ne 0 ]; then
14
- echo "Please review any new additions to the text files under 'identifiers/'."
15
- echo "If any of these names are close to an existing name, update the offending"
16
- echo "fingerprint to use the existing name instead. Once the fingerprints are fixed,"
17
- echo "remove the 'extra' names from the identifiers files, and run the tool again."
18
- exit 1
19
- fi
20
-
21
- exit 0
data/update_cpes.py DELETED
@@ -1,250 +0,0 @@
1
- #!/usr/bin/env python
2
-
3
- import logging
4
- import re
5
- import sys
6
-
7
- import yaml
8
- from lxml import etree
9
-
10
- def parse_r7_remapping(file):
11
- with open(file) as remap_file:
12
- return yaml.safe_load(remap_file)["mappings"]
13
-
14
- def parse_cpe_vp_map(file):
15
- vp_map = {} # cpe_type -> vendor -> products
16
- parser = etree.XMLParser(remove_comments=False)
17
- doc = etree.parse(file, parser)
18
- namespaces = {'ns': 'http://cpe.mitre.org/dictionary/2.0', 'meta': 'http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2'}
19
- for entry in doc.xpath("//ns:cpe-list/ns:cpe-item", namespaces=namespaces):
20
- cpe_name = entry.get("name")
21
- if not cpe_name:
22
- continue
23
-
24
- # If the entry is deprecated then don't add it to our list of valid CPEs.
25
- if entry.get("deprecated"):
26
- continue
27
-
28
- cpe_match = re.match('^cpe:/([aho]):([^:]+):([^:]+)', cpe_name)
29
-
30
- if cpe_match:
31
- cpe_type, vendor, product = cpe_match.group(1, 2, 3)
32
- if cpe_type not in vp_map:
33
- vp_map[cpe_type] = {}
34
- if vendor not in vp_map[cpe_type]:
35
- vp_map[cpe_type][vendor] = set()
36
- product = product.replace('%2f', '/')
37
- vp_map[cpe_type][vendor].add(product)
38
- else:
39
- logging.error("Unexpected CPE %s", cpe_name)
40
-
41
- return vp_map
42
-
43
- def main():
44
- if len(sys.argv) != 4:
45
- logging.critical("Expecting exactly 3 arguments; recog XML file, CPE 2.3 XML dictionary, JSON remapping, got %s", (len(sys.argv) - 1))
46
- sys.exit(1)
47
-
48
- cpe_vp_map = parse_cpe_vp_map(sys.argv[2])
49
- if not cpe_vp_map:
50
- logging.critical("No CPE vendor => product mappings read from CPE 2.3 XML dictionary %s", sys.argv[2])
51
- sys.exit(1)
52
-
53
- r7_vp_map = parse_r7_remapping(sys.argv[3])
54
- if not r7_vp_map:
55
- logging.warning("No Rapid7 vendor/product => CPE mapping read from %s", sys.argv[3])
56
-
57
- update_cpes(sys.argv[1], cpe_vp_map, r7_vp_map)
58
-
59
- def lookup_cpe(vendor, product, cpe_type, cpe_table, remap):
60
- """Identify the correct vendor and product values for a CPE
61
-
62
- This function attempts to determine the correct CPE using vendor and product
63
- values supplied by the caller as well as a remapping dictionary for mapping
64
- these values to more correct values used by NIST.
65
-
66
- For example, the remapping might tell us that a value of 'alpine' for the
67
- vendor string should be 'alpinelinux' instead, or for product 'solaris'
68
- should be 'sunos'.
69
-
70
- This function should only emit values seen in the official NIST CPE list
71
- which is provided to it in cpe_table.
72
-
73
- Lookup priority:
74
- 1. Original vendor / product
75
- 2. Original vendor / remap product
76
- 3. Remap vendor / original product
77
- 4. Remap vendor / remap product
78
-
79
- Args:
80
- vendor (str): vendor name
81
- product (str): product name
82
- cpe_type (str): CPE type - o, a, h, etc.
83
- cpe_table (dict): dict containing the official NIST CPE data
84
- remap (dict): dict containing the remapping values
85
- Returns:
86
- success, vendor, product
87
- """
88
-
89
- if (
90
- vendor in cpe_table[cpe_type]
91
- and product in cpe_table[cpe_type][vendor]
92
- ):
93
- # Hot path, success with original values
94
- return True, vendor, product
95
-
96
- # Everything else depends on a remap of some sort.
97
- # get the remappings for this one vendor string.
98
- vendor_remap = None
99
-
100
- remap_type = remap.get(cpe_type, None)
101
- if remap_type:
102
- vendor_remap = remap_type.get(vendor, None)
103
-
104
- if vendor_remap:
105
- # If we have product remappings, work that angle next
106
- possible_product = None
107
- if (
108
- vendor_remap.get('products', None)
109
- and product in vendor_remap['products']
110
- ):
111
- possible_product = vendor_remap['products'][product]
112
-
113
- if (vendor in cpe_table[cpe_type]
114
- and possible_product
115
- and possible_product in cpe_table[cpe_type][vendor]):
116
- # Found original vendor, remap product
117
- return True, vendor, possible_product
118
-
119
- # Start working the process to find a match with a remapped vendor name
120
- if vendor_remap.get('vendor', None):
121
- new_vendor = vendor_remap['vendor']
122
-
123
- if new_vendor in cpe_table[cpe_type]:
124
-
125
- if product in cpe_table[cpe_type][new_vendor]:
126
- # Found remap vendor, original product
127
- return True, new_vendor, product
128
-
129
- if possible_product and possible_product in cpe_table[cpe_type][new_vendor]:
130
- # Found remap vendor, remap product
131
- return True, new_vendor, possible_product
132
-
133
-
134
- logging.error("Product %s from vendor %s invalid for CPE %s and no mapping",
135
- product, vendor, cpe_type)
136
- return False, None, None
137
-
138
-
139
- def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
140
- parser = etree.XMLParser(remove_comments=False, remove_blank_text=True)
141
- doc = etree.parse(xml_file, parser)
142
-
143
- for fingerprint in doc.xpath('//fingerprint'):
144
-
145
- # collect all the params, grouping by os and service params that could be used to compute a CPE
146
- params = {}
147
- for param in fingerprint.xpath('./param'):
148
- name = param.attrib['name']
149
- # remove any existing CPE params
150
- if re.match(r'^.*\.cpe\d{0,2}$', name):
151
- param.getparent().remove(param)
152
- continue
153
-
154
- match = re.search(r'^(?P<fp_type>hw|os|service(?:\.component)?)\.', name)
155
- if match:
156
- fp_type = match.group('fp_type')
157
- if not fp_type in params:
158
- params[fp_type] = {}
159
- if name in params[fp_type]:
160
- raise ValueError('Duplicated fingerprint named {} in fingerprint {} in file {}'.format(name, fingerprint.attrib['pattern'], xml_file))
161
- params[fp_type][name] = param
162
-
163
-
164
- # for each of the applicable os/service param groups, build a CPE
165
- for fp_type in params:
166
- if fp_type == 'os':
167
- cpe_type = 'o'
168
- elif fp_type.startswith('service'):
169
- cpe_type = 'a'
170
- elif fp_type == 'hw':
171
- cpe_type = 'h'
172
- else:
173
- raise ValueError('Unhandled param type {}'.format(fp_type))
174
-
175
- # extract the vendor/product/version values from each os/service group,
176
- # using the static value ('Apache', for example) when pos is 0, and
177
- # otherwise use a value that contains interpolation markers such that
178
- # products/projects that use recog content can insert the value
179
- # extracted from the banner/other data via regex capturing groups
180
- fp_data = {
181
- 'vendor': None,
182
- 'product': None,
183
- 'version': '-',
184
- }
185
- for fp_datum in fp_data:
186
- fp_datum_param_name = "{}.{}".format(fp_type, fp_datum)
187
- if fp_datum_param_name in params[fp_type]:
188
- fp_datum_e = params[fp_type][fp_datum_param_name]
189
- if fp_datum_e.attrib['pos'] == '0':
190
- fp_data[fp_datum] = fp_datum_e.attrib['value']
191
- else:
192
- fp_data[fp_datum] = "{{{}}}".format(fp_datum_e.attrib['name'])
193
-
194
- vendor = fp_data['vendor']
195
- product = fp_data['product']
196
- version = fp_data['version']
197
-
198
- # build a reasonable looking CPE value from the vendor/product/version,
199
- # lowercasing, replacing whitespace with _, and more
200
- if vendor and product:
201
- if not cpe_type in cpe_vp_map:
202
- logging.error("Didn't find CPE type '%s' for '%s' '%s'", cpe_type, vendor, product)
203
- continue
204
-
205
- vendor = vendor.lower().replace(' ', '_').replace(',', '')
206
- product = product.lower().replace(' ', '_').replace(',', '').replace('!', '%21')
207
- if 'unknown' in [vendor, product]:
208
- continue
209
-
210
- if (vendor.startswith('{') and vendor.endswith('}')) or (product.startswith('{') and product.endswith('}')):
211
- continue
212
-
213
- success, vendor, product = lookup_cpe(vendor, product, cpe_type, cpe_vp_map, r7_vp_map)
214
- if not success:
215
- continue
216
-
217
- # Sanity check the value to ensure that no invalid values will
218
- # slip in due to logic or mapping bugs.
219
- # If it's not in the official NIST list then log it and kick it out
220
- if product not in cpe_vp_map[cpe_type][vendor]:
221
- logging.error("Invalid CPE type %s created for vendor %s and product %s. This may be due to an invalid mapping.", cpe_type, vendor, product)
222
- continue
223
-
224
- # building the CPE string
225
- # Last minute escaping of '/' and `!`
226
- product = product.replace('/', '\/').replace('%21', '\!')
227
- cpe_value = 'cpe:/{}:{}:{}'.format(cpe_type, vendor, product)
228
-
229
- if version:
230
- cpe_value += ":{}".format(version)
231
-
232
- cpe_param = etree.Element('param')
233
- cpe_param.attrib['pos'] = '0'
234
- cpe_param.attrib['name'] = '{}.cpe23'.format(fp_type)
235
- cpe_param.attrib['value'] = cpe_value
236
-
237
- for param_name in params[fp_type]:
238
- param = params[fp_type][param_name]
239
- parent = param.getparent()
240
- index = parent.index(param) + 1
241
- parent.insert(index, cpe_param)
242
-
243
- root = doc.getroot()
244
-
245
- with open(xml_file, 'wb') as xml_out:
246
- xml_out.write(etree.tostring(root, pretty_print=True, xml_declaration=True, encoding=doc.docinfo.encoding))
247
-
248
- if __name__ == '__main__':
249
- try: sys.exit(main())
250
- except KeyboardInterrupt: pass