recog 2.0.24 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/features/data/matching_banners_fingerprints.xml +2 -1
- data/features/data/multiple_banners_fingerprints.xml +2 -0
- data/features/match.feature +6 -6
- data/lib/recog/db.rb +33 -11
- data/lib/recog/db_manager.rb +6 -2
- data/lib/recog/fingerprint.rb +39 -2
- data/lib/recog/nizer.rb +93 -20
- data/lib/recog/version.rb +1 -1
- data/spec/lib/fingerprint_self_test_spec.rb +7 -0
- data/spec/lib/recog/nizer_spec.rb +165 -3
- data/xml/apache_os.xml +1 -1
- data/xml/architecture.xml +1 -1
- data/xml/fingerprints.xsd +91 -0
- data/xml/ftp_banners.xml +456 -74
- data/xml/h323_callresp.xml +1 -1
- data/xml/hp_pjl_id.xml +4 -1
- data/xml/http_cookies.xml +1 -1
- data/xml/http_servers.xml +1 -1
- data/xml/http_wwwauth.xml +1 -1
- data/xml/imap_banners.xml +1 -1
- data/xml/ldap_searchresult.xml +1 -1
- data/xml/mdns_device-info_txt.xml +1 -1
- data/xml/mdns_workstation_txt.xml +1 -1
- data/xml/mysql_banners.xml +1 -1
- data/xml/mysql_error.xml +1 -1
- data/xml/nntp_banners.xml +1 -1
- data/xml/ntp_banners.xml +1 -1
- data/xml/operating_system.xml +1 -1
- data/xml/pop_banners.xml +1 -1
- data/xml/rsh_resp.xml +1 -1
- data/xml/sip_banners.xml +1 -1
- data/xml/sip_user_agents.xml +1 -1
- data/xml/smb_native_lm.xml +1 -1
- data/xml/smb_native_os.xml +1 -1
- data/xml/smtp_banners.xml +5 -1
- data/xml/smtp_debug.xml +4 -1
- data/xml/smtp_ehlo.xml +4 -1
- data/xml/smtp_expn.xml +4 -1
- data/xml/smtp_help.xml +4 -1
- data/xml/smtp_mailfrom.xml +1 -1
- data/xml/smtp_noop.xml +4 -1
- data/xml/smtp_quit.xml +4 -1
- data/xml/smtp_rcptto.xml +1 -1
- data/xml/smtp_rset.xml +4 -1
- data/xml/smtp_turn.xml +4 -1
- data/xml/smtp_vrfy.xml +4 -1
- data/xml/snmp_sysdescr.xml +1 -1
- data/xml/snmp_sysobjid.xml +1 -1
- data/xml/ssh_banners.xml +1 -1
- data/xml/upnp_banners.xml +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11aff7826c38e379d719a70b39406f4d861f8d3f
|
4
|
+
data.tar.gz: 98f54e902871cb73ddcffbb7e0f72be5b21bf143
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 398dc88dfda3030d5d0e8da3c0774fc95ec99c2506f9f15ba528ef5ec930591f02f43d313ec5ed06cb4b7bb28798ef71bf53a539a6b50b8158a7f0b69877e0ed
|
7
|
+
data.tar.gz: 19bf9ddccb4031ade9bf5d94feb4516df400816eaf81a894b24716e06ef7e5d5a2400bd0424d55875b38451fcad83edb7f99f7b00ac1e38fa059447682f17d0a
|
data/README.md
CHANGED
@@ -75,7 +75,7 @@ Matches can be tested on the command-line in a similar fashion:
|
|
75
75
|
|
76
76
|
```
|
77
77
|
$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
|
78
|
-
MATCH: {"service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
78
|
+
MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
79
79
|
```
|
80
80
|
|
81
81
|
### Best Practices
|
@@ -1,5 +1,5 @@
|
|
1
1
|
<?xml version="1.0"?>
|
2
|
-
<fingerprints>
|
2
|
+
<fingerprints protocol="ftp" database_type="service">
|
3
3
|
<fingerprint pattern="^-{10} Welcome to Pure-FTPd (.*)-{10}$">
|
4
4
|
<example>---------- Welcome to Pure-FTPd ----------</example>
|
5
5
|
<description>Pure-FTPd
|
@@ -8,6 +8,7 @@
|
|
8
8
|
<param pos="1" name="pureftpd.config"/>
|
9
9
|
<param pos="0" name="service.family" value="Pure-FTPd"/>
|
10
10
|
<param pos="0" name="service.product" value="Pure-FTPd"/>
|
11
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
11
12
|
</fingerprint>
|
12
13
|
<fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
|
13
14
|
<description>SunOS/Solaris</description>
|
@@ -16,10 +16,12 @@
|
|
16
16
|
<param pos="1" name="pureftpd.config"/>
|
17
17
|
<param pos="0" name="service.family" value="Pure-FTPd"/>
|
18
18
|
<param pos="0" name="service.product" value="Pure-FTPd"/>
|
19
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
19
20
|
</fingerprint>
|
20
21
|
<fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
|
21
22
|
<description>SunOS/Solaris</description>
|
22
23
|
<example>example.com FTP server (SunOS 5.7) ready.</example>
|
24
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
23
25
|
<param pos="0" name="os.vendor" value="Sun"/>
|
24
26
|
<param pos="0" name="os.family" value="Solaris"/>
|
25
27
|
<param pos="0" name="os.product" value="Solaris"/>
|
data/features/match.feature
CHANGED
@@ -3,8 +3,8 @@ Feature: Match
|
|
3
3
|
When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
|
4
4
|
Then it should pass with:
|
5
5
|
"""
|
6
|
-
MATCH: {"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
7
|
-
MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
6
|
+
MATCH: {"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
7
|
+
MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
8
8
|
"""
|
9
9
|
|
10
10
|
Scenario: Fails at finding matches
|
@@ -19,14 +19,14 @@ Feature: Match
|
|
19
19
|
When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
|
20
20
|
Then it should pass with:
|
21
21
|
"""
|
22
|
-
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"},{"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
23
|
-
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
22
|
+
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"},{"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "service.protocol"=>"ftp", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
23
|
+
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
24
24
|
"""
|
25
25
|
|
26
26
|
Scenario: Finds first matches using no-multi-match flag
|
27
27
|
When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
|
28
28
|
Then it should pass with:
|
29
29
|
"""
|
30
|
-
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
31
|
-
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
30
|
+
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
31
|
+
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
32
32
|
"""
|
data/lib/recog/db.rb
CHANGED
@@ -13,13 +13,34 @@ class DB
|
|
13
13
|
# against strings that make sense for the {#match_key}
|
14
14
|
attr_reader :fingerprints
|
15
15
|
|
16
|
-
# @return [String] Taken from the `fingerprints/matches`
|
16
|
+
# @return [String] Taken from the `fingerprints/matches` attribute, or
|
17
17
|
# defaults to the basename of {#path} without the `.xml` extension.
|
18
18
|
attr_reader :match_key
|
19
19
|
|
20
|
+
# @return [String] Taken from the `fingerprints/protocol` attribute, or
|
21
|
+
# defaults to an empty string
|
22
|
+
attr_reader :protocol
|
23
|
+
|
24
|
+
# @return [String] Taken from the `fingerprints/database_type` attribute
|
25
|
+
# defaults to an empty string
|
26
|
+
attr_reader :database_type
|
27
|
+
|
28
|
+
# @return [Float] Taken from the `fingerprints/preference` attribute,
|
29
|
+
# defaults to 0.10. Used when ordering databases, highest numbers
|
30
|
+
# are given priority and are processed first.
|
31
|
+
attr_reader :preference
|
32
|
+
|
33
|
+
# Default Fingerprint database preference when it isn't specified in file
|
34
|
+
# Do not use a value below 0.10 so as to allow users to specify lower
|
35
|
+
# values in their own custom XML that will always run last.
|
36
|
+
DEFAULT_FP_PREFERENCE = 0.10
|
37
|
+
|
20
38
|
# @param path [String]
|
21
39
|
def initialize(path)
|
22
40
|
@match_key = nil
|
41
|
+
@protocol = ''
|
42
|
+
@database_type = ''
|
43
|
+
@preference = DEFAULT_FP_PREFERENCE.to_f
|
23
44
|
@path = path
|
24
45
|
@fingerprints = []
|
25
46
|
|
@@ -30,24 +51,25 @@ class DB
|
|
30
51
|
def parse_fingerprints
|
31
52
|
xml = nil
|
32
53
|
|
33
|
-
File.open(self.path,
|
54
|
+
File.open(self.path, 'rb') do |fd|
|
34
55
|
xml = Nokogiri::XML(fd.read(fd.stat.size))
|
35
56
|
end
|
36
57
|
|
37
58
|
raise "#{self.path} is invalid XML: #{xml.errors.join(',')}" unless xml.errors.empty?
|
38
59
|
|
39
|
-
xml.xpath(
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
60
|
+
xml.xpath('/fingerprints').each do |fbase|
|
61
|
+
|
62
|
+
@match_key = fbase['matches'].to_s if fbase['matches']
|
63
|
+
@protocol = fbase['protocol'].to_s if fbase['protocol']
|
64
|
+
@database_type = fbase['database_type'].to_s if fbase['database_type']
|
65
|
+
@preference = fbase['preference'].to_f if fbase['preference']
|
44
66
|
|
45
|
-
unless @match_key
|
46
|
-
@match_key = File.basename(self.path).sub(/\.xml$/, '')
|
47
67
|
end
|
48
68
|
|
49
|
-
|
50
|
-
|
69
|
+
@match_key = File.basename(self.path).sub(/\.xml$/, '') unless @match_key
|
70
|
+
|
71
|
+
xml.xpath('/fingerprints/fingerprint').each do |fprint|
|
72
|
+
@fingerprints << Fingerprint.new(fprint, @match_key, @protocol)
|
51
73
|
end
|
52
74
|
|
53
75
|
xml = nil
|
data/lib/recog/db_manager.rb
CHANGED
@@ -13,8 +13,12 @@ class DBManager
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def load_databases
|
16
|
-
|
17
|
-
self.
|
16
|
+
if File.directory?(self.path)
|
17
|
+
Dir[self.path + "/*.xml"].each do |dbxml|
|
18
|
+
self.databases << DB.new(dbxml)
|
19
|
+
end
|
20
|
+
else
|
21
|
+
self.databases << DB.new(self.path)
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
data/lib/recog/fingerprint.rb
CHANGED
@@ -27,23 +27,49 @@ class Fingerprint
|
|
27
27
|
attr_reader :tests
|
28
28
|
|
29
29
|
# @param xml [Nokogiri::XML::Element]
|
30
|
-
|
30
|
+
# @param match_key [String] See Recog::DB
|
31
|
+
# @param protocol [String] Protocol such as ftp, mssql, http, etc.
|
32
|
+
def initialize(xml, match_key=nil, protocol=nil)
|
33
|
+
@match_key = match_key
|
34
|
+
@protocol = protocol
|
31
35
|
@name = parse_description(xml)
|
32
36
|
@regex = create_regexp(xml)
|
33
37
|
@params = {}
|
34
38
|
@tests = []
|
35
39
|
|
40
|
+
@protocol.downcase! if @protocol
|
36
41
|
parse_examples(xml)
|
37
42
|
parse_params(xml)
|
38
43
|
end
|
39
44
|
|
45
|
+
def output_diag_data(message, data, exception)
|
46
|
+
STDERR.puts message
|
47
|
+
STDERR.puts exception.inspect
|
48
|
+
STDERR.puts "Length: #{data.length}"
|
49
|
+
STDERR.puts "Encoding: #{data.encoding}"
|
50
|
+
STDERR.puts "Problematic data:\n#{data}"
|
51
|
+
STDERR.puts "Raw bytes:\n#{data.pretty_inspect}\n"
|
52
|
+
end
|
53
|
+
|
40
54
|
# Attempt to match the given string.
|
41
55
|
#
|
42
56
|
# @param match_string [String]
|
43
57
|
# @return [Hash,nil] Keys will be host, service, and os attributes
|
44
58
|
def match(match_string)
|
45
59
|
# match_string.force_encoding('BINARY') if match_string
|
46
|
-
|
60
|
+
begin
|
61
|
+
match_data = @regex.match(match_string)
|
62
|
+
rescue Encoding::CompatibilityError => e
|
63
|
+
begin
|
64
|
+
# Replace invalid UTF-8 characters with spaces, just as DAP does.
|
65
|
+
encoded_str = match_string.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
|
66
|
+
match_data = @regex.match(encoded_str)
|
67
|
+
rescue Exception => e
|
68
|
+
output_diag_data('Exception while re-encoding match_string to UTF-8', match_string, e)
|
69
|
+
end
|
70
|
+
rescue Exception => e
|
71
|
+
output_diag_data('Exception while running regex against match_string', match_string, e)
|
72
|
+
end
|
47
73
|
return if match_data.nil?
|
48
74
|
|
49
75
|
result = { 'matched' => @name }
|
@@ -58,6 +84,17 @@ class Fingerprint
|
|
58
84
|
result[k] = match_data[ pos ]
|
59
85
|
end
|
60
86
|
end
|
87
|
+
|
88
|
+
# Use the protocol specified in the XML database if there isn't one
|
89
|
+
# provided as part of this fingerprint.
|
90
|
+
if @protocol
|
91
|
+
unless result['service.protocol']
|
92
|
+
result['service.protocol'] = @protocol
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
result['fingerprint_db'] = @match_key if @match_key
|
97
|
+
|
61
98
|
return result
|
62
99
|
end
|
63
100
|
|
data/lib/recog/nizer.rb
CHANGED
@@ -20,43 +20,116 @@ class Nizer
|
|
20
20
|
}
|
21
21
|
|
22
22
|
@@db_manager = nil
|
23
|
+
@@db_sorted = false
|
24
|
+
|
25
|
+
|
26
|
+
#
|
27
|
+
# Load fingerprints from a specific file or directory
|
28
|
+
# This will not preserve any fingerprints that have already been loaded
|
29
|
+
# @param path [String] Path to file or directory of XML fingerprints
|
30
|
+
def self.load_db(path = nil)
|
31
|
+
if path
|
32
|
+
@@db_manager = Recog::DBManager.new(path)
|
33
|
+
else
|
34
|
+
@@db_manager = Recog::DBManager.new
|
35
|
+
end
|
36
|
+
|
37
|
+
# Sort the databases, no behavior or result change for those calling
|
38
|
+
# Nizer.match or Nizer.multi_match as they have a single DB
|
39
|
+
@@db_manager.databases.sort! { |a, b| b.preference <=> a.preference }
|
40
|
+
@@db_sorted = true
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Destroy the current DBManager object
|
45
|
+
def self.unload_db
|
46
|
+
@@db_manager = nil
|
47
|
+
@@db_sorted = false
|
48
|
+
end
|
23
49
|
|
50
|
+
#
|
51
|
+
# Display the fingerprint databases in the order in which they will be used
|
52
|
+
# to match banners. This is useful for fingerprint tuning and debugging.
|
53
|
+
def self.display_db_order
|
54
|
+
self.load_db unless @@db_manager
|
55
|
+
|
56
|
+
puts format('%s %-22s %-8s %s', 'Preference', 'Database', 'Type', 'Protocol')
|
57
|
+
@@db_manager.databases.each do |db|
|
58
|
+
puts format('%10.3f %-22s %-8s %s', db.preference, db.match_key,
|
59
|
+
db.database_type, db.protocol)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
#
|
64
|
+
# 2016.11 - Rewritten to be wrapper around #match_db_all, functionality
|
65
|
+
# and results must remain unchanged.
|
24
66
|
#
|
25
67
|
# Locate a database that corresponds with the `match_key` and attempt to
|
26
|
-
# find a matching {Fingerprint fingerprint}, stopping at the first hit.
|
27
|
-
# when no matching database or fingerprint is found.
|
68
|
+
# find a matching {Fingerprint fingerprint}, stopping at the first hit.
|
69
|
+
# Returns `nil` when no matching database or fingerprint is found.
|
28
70
|
#
|
29
71
|
# @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
|
30
|
-
# @
|
72
|
+
# @param match_string [String] String to match
|
73
|
+
# @return (see Fingerprint#match) or nil
|
31
74
|
def self.match(match_key, match_string)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
db.fingerprints.each do |fprint|
|
37
|
-
m = fprint.match(match_string)
|
38
|
-
return m if m
|
39
|
-
end
|
40
|
-
end
|
41
|
-
nil
|
75
|
+
filter = { match_key: match_key, multi_match: false }
|
76
|
+
matches = self.match_all_db(match_string, filter)
|
77
|
+
|
78
|
+
matches[0]
|
42
79
|
end
|
43
80
|
|
81
|
+
#
|
82
|
+
# @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
|
83
|
+
# @param match_string [String] String to match
|
84
|
+
# @return [Array] Array of Fingerprint#match or empty array
|
44
85
|
def self.multi_match(match_key, match_string)
|
45
|
-
|
46
|
-
|
86
|
+
filter = { match_key: match_key, multi_match: true }
|
87
|
+
self.match_all_db(match_string, filter)
|
88
|
+
end
|
47
89
|
|
48
|
-
|
90
|
+
#
|
91
|
+
# Search all fingerprint dbs and attempt to find matching
|
92
|
+
# {Fingerprint fingerprint}s. It will return the first match found
|
93
|
+
# unless the :multi_match option is used to request all matches.
|
94
|
+
# Returns an array of all matching fingerprints or an empty array.
|
95
|
+
#
|
96
|
+
# @param match_string [String] Service banner to match
|
97
|
+
# @param [Hash] filters This hash contains filters used to limit the
|
98
|
+
# results to just those from specific types of fingerprints.
|
99
|
+
# The values that these filters match come from the 'fingerprints' top
|
100
|
+
# level element in the fingerprint DB XML or, in the case of 'protocol',
|
101
|
+
# this value can be overridden at the individual fingerprint level by
|
102
|
+
# setting a value for 'service.protocol'
|
103
|
+
#
|
104
|
+
# With the exception of 'match_key', the filters below match the
|
105
|
+
# 'fingerprints' attributes with the same name.
|
106
|
+
# @option filters [String] :match_key Value from XML 'matches' or file name
|
107
|
+
# @option filters [String] :database_type fprint db type: service, util.os, etc.
|
108
|
+
# @option filters [String] :protocol Protocol (ftp, smtp, etc.)
|
109
|
+
# @option filters [Boolean] :multi_match Return all matches instead of first
|
110
|
+
# @return [Array] Array of Fingerprint#match or empty array
|
111
|
+
def self.match_all_db(match_string, filters = {})
|
112
|
+
match_string = match_string.to_s.unpack('C*').pack('C*')
|
113
|
+
matches = Array.new # array to hold all fingerprint matches
|
114
|
+
|
115
|
+
self.load_db unless @@db_manager
|
49
116
|
|
50
117
|
@@db_manager.databases.each do |db|
|
51
|
-
next
|
52
|
-
|
118
|
+
next if filters[:match_key] && !filters[:match_key].eql?(db.match_key)
|
119
|
+
next if filters[:database_type] && !filters[:database_type].eql?(db.database_type)
|
53
120
|
db.fingerprints.each do |fp|
|
54
121
|
m = fp.match(match_string)
|
55
|
-
|
122
|
+
if m
|
123
|
+
# Filter on protocol after match since each individual fp
|
124
|
+
# can contain its own 'protocol' value that overrides the
|
125
|
+
# one set at the DB level.
|
126
|
+
matches.push(m) unless filters[:protocol] && !filters[:protocol].eql?(m['service.protocol'])
|
127
|
+
return matches unless filters[:multi_match]
|
128
|
+
end
|
56
129
|
end
|
57
130
|
end
|
58
131
|
|
59
|
-
|
132
|
+
matches
|
60
133
|
end
|
61
134
|
|
62
135
|
#
|
data/lib/recog/version.rb
CHANGED
@@ -21,6 +21,13 @@ describe Recog::DB do
|
|
21
21
|
expect(db.match_key).not_to be_empty
|
22
22
|
end
|
23
23
|
|
24
|
+
it "has valid 'preference' value" do
|
25
|
+
# Reserve values below 0.10 and above 0.90 for users
|
26
|
+
# See xml/fingerprints.xsd
|
27
|
+
expect(db.preference.class).to be ::Float
|
28
|
+
expect(db.preference).to be_between(0.10, 0.90)
|
29
|
+
end
|
30
|
+
|
24
31
|
db.fingerprints.each_index do |i|
|
25
32
|
fp = db.fingerprints[i]
|
26
33
|
|
@@ -1,6 +1,12 @@
|
|
1
1
|
require 'recog'
|
2
2
|
require 'yaml'
|
3
3
|
|
4
|
+
|
5
|
+
VALID_FILTER = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'util.os'}
|
6
|
+
NOMATCH_MATCH_KEY = {match_key: 'no_such_987', protocol: 'smb', database_type: 'util.os'}
|
7
|
+
NOMATCH_PROTO = {match_key: 'smb.native_os', protocol: 'no_such_987', database_type: 'util.os'}
|
8
|
+
NOMATCH_TYPE = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'no_such_987'}
|
9
|
+
|
4
10
|
describe Recog::Nizer do
|
5
11
|
subject { described_class }
|
6
12
|
|
@@ -24,6 +30,15 @@ describe Recog::Nizer do
|
|
24
30
|
end
|
25
31
|
end
|
26
32
|
|
33
|
+
let(:nomatch_result) { subject.match('smb.native_os', 'no_such_987_76tgklh') }
|
34
|
+
it "returns a nil when data cannot be matched" do
|
35
|
+
expect(nomatch_result).to be_nil
|
36
|
+
end
|
37
|
+
|
38
|
+
let(:invalid_db_result) { subject.match('no_such_987', data) }
|
39
|
+
it "returns a nil when match_key search doesn't match" do
|
40
|
+
expect(invalid_db_result).to be_nil
|
41
|
+
end
|
27
42
|
end
|
28
43
|
end
|
29
44
|
|
@@ -36,6 +51,83 @@ describe Recog::Nizer do
|
|
36
51
|
end
|
37
52
|
end
|
38
53
|
|
54
|
+
describe ".match_all_db" do
|
55
|
+
File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
|
56
|
+
data = line.strip
|
57
|
+
context "with smb_native_os:#{data}" do
|
58
|
+
let(:match_all_result) { subject.match_all_db(data, VALID_FILTER) }
|
59
|
+
|
60
|
+
it "returns an array" do
|
61
|
+
expect(match_all_result.class).to eq(::Array)
|
62
|
+
end
|
63
|
+
|
64
|
+
it "returns a successful match" do
|
65
|
+
expect(match_all_result[0]['matched']).to match(/^[A-Z]/)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "correctly matches service or os" do
|
69
|
+
if data =~ /^Windows/
|
70
|
+
expect(match_all_result[0]['os.product']).to match(/^Windows/)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
it "correctly matches protocol" do
|
75
|
+
expect(match_all_result[0]['service.protocol']).to eq('smb')
|
76
|
+
end
|
77
|
+
|
78
|
+
let(:no_filter_result) { subject.match_all_db(data) }
|
79
|
+
it "returns an array when searching without a filter" do
|
80
|
+
expect(no_filter_result.class).to eq(::Array)
|
81
|
+
end
|
82
|
+
|
83
|
+
it "returns a successful match when searching without a filter" do
|
84
|
+
expect(no_filter_result[0]['matched']).to match(/^[A-Z]/)
|
85
|
+
end
|
86
|
+
|
87
|
+
it "correctly matches service or os when searching without a filter" do
|
88
|
+
if data =~ /^Windows/
|
89
|
+
expect(no_filter_result[0]['os.product']).to match(/^Windows/)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
let(:nomatch_db_result) { subject.match_all_db(data, NOMATCH_MATCH_KEY) }
|
94
|
+
it "returns an array when match_key search doesn't match" do
|
95
|
+
expect(nomatch_db_result.class).to eq(::Array)
|
96
|
+
end
|
97
|
+
it "returns an empty array when match_key search doesn't match" do
|
98
|
+
expect(nomatch_db_result).to be_empty
|
99
|
+
end
|
100
|
+
|
101
|
+
let(:nomatch_proto_result) { subject.match_all_db(data, NOMATCH_PROTO) }
|
102
|
+
it "returns an array when protocol search doesn't match" do
|
103
|
+
expect(nomatch_proto_result.class).to eq(::Array)
|
104
|
+
end
|
105
|
+
it "returns an empty array when protocol search doesn't match" do
|
106
|
+
expect(nomatch_proto_result).to be_empty
|
107
|
+
end
|
108
|
+
|
109
|
+
let(:nomatch_type_result) { subject.match_all_db(data, NOMATCH_TYPE) }
|
110
|
+
it "returns an array when database_type search doesn't match" do
|
111
|
+
expect(nomatch_type_result.class).to eq(::Array)
|
112
|
+
end
|
113
|
+
it "returns an empty array when database_type search doesn't match" do
|
114
|
+
expect(nomatch_proto_result).to be_empty
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
line = 'non-existent'
|
120
|
+
context "with non-existent match" do
|
121
|
+
let(:match_result) {subject.match_all_db(line) }
|
122
|
+
it "returns an array" do
|
123
|
+
expect(match_result.class).to eq(::Array)
|
124
|
+
end
|
125
|
+
it "returns an empty array" do
|
126
|
+
expect(match_result).to be_empty
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
39
131
|
describe ".multi_match" do
|
40
132
|
File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
|
41
133
|
data = line.strip
|
@@ -58,8 +150,46 @@ describe Recog::Nizer do
|
|
58
150
|
end
|
59
151
|
end
|
60
152
|
end
|
153
|
+
|
154
|
+
let(:invalid_db_result) { subject.multi_match('no_such_987', data) }
|
155
|
+
it "returns an array when passed an invalid match_key" do
|
156
|
+
expect(invalid_db_result.class).to eq(::Array)
|
157
|
+
end
|
158
|
+
|
159
|
+
it "returns an empty array when passed an invalid match_key" do
|
160
|
+
expect(invalid_db_result).to be_empty
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
end
|
165
|
+
|
166
|
+
data = 'Windows Server 2012 R2 Standard 9600'
|
167
|
+
context "with {data}" do
|
168
|
+
let(:match_results) {subject.multi_match('smb.native_os', data) }
|
169
|
+
|
170
|
+
it "returns an array" do
|
171
|
+
expect(match_results.class).to eq(::Array)
|
172
|
+
end
|
173
|
+
|
174
|
+
it "returns at least two successful matches" do
|
175
|
+
expect(match_results.size).to be > 1
|
176
|
+
end
|
177
|
+
|
178
|
+
it "correctly matches os.product for all matches" do
|
179
|
+
match_results do |mr|
|
180
|
+
if data =~ /^Windows/
|
181
|
+
expect(mr['os.product']).to match(/^Windows/)
|
182
|
+
end
|
183
|
+
end
|
61
184
|
end
|
62
185
|
|
186
|
+
it "correctly matches protocol for all matches" do
|
187
|
+
match_results do |mr|
|
188
|
+
if data =~ /^Windows/
|
189
|
+
expect(mr['service.protocol']).to eq('smb')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
63
193
|
end
|
64
194
|
|
65
195
|
line = 'non-existent'
|
@@ -77,7 +207,6 @@ describe Recog::Nizer do
|
|
77
207
|
end
|
78
208
|
|
79
209
|
describe ".best_os_match" do
|
80
|
-
|
81
210
|
# Demonstrates how this method picks up additional attributes from other members of the winning
|
82
211
|
# os.product match group and applies them to the result.
|
83
212
|
matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_1.yml'))))
|
@@ -134,8 +263,7 @@ describe Recog::Nizer do
|
|
134
263
|
|
135
264
|
end
|
136
265
|
|
137
|
-
|
138
|
-
|
266
|
+
describe ".best_service_match" do
|
139
267
|
# Demonstrates how this method picks up additional attributes from other members of the winning
|
140
268
|
# service.product match group and applies them to the result.
|
141
269
|
matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_service_match_1.yml'))))
|
@@ -165,4 +293,38 @@ describe Recog::Nizer do
|
|
165
293
|
|
166
294
|
end
|
167
295
|
|
296
|
+
|
297
|
+
describe '.load_db' do
|
298
|
+
file_path = File.expand_path(File.join('spec', 'data', 'test_fingerprints.xml'))
|
299
|
+
context "with #{file_path}" do
|
300
|
+
let(:fp_db) { subject.load_db(file_path) }
|
301
|
+
it "loads without error" do
|
302
|
+
expect(fp_db).to be true
|
303
|
+
subject.unload_db()
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
context "with no path specified" do
|
308
|
+
let(:fp_db) { subject.load_db }
|
309
|
+
it "loads without error" do
|
310
|
+
expect(fp_db).to be true
|
311
|
+
subject.unload_db()
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
context "with empty file path" do
|
316
|
+
it "raises an error" do
|
317
|
+
expect { subject.load_db('') }.to raise_error(Errno::ENOENT)
|
318
|
+
subject.unload_db()
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
context "with invalid file path" do
|
323
|
+
it "raises an error" do
|
324
|
+
expect { subject.load_db('no_such_987_file_path') }.to raise_error(Errno::ENOENT)
|
325
|
+
subject.unload_db()
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
168
330
|
end
|