recog 2.0.24 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/features/data/matching_banners_fingerprints.xml +2 -1
- data/features/data/multiple_banners_fingerprints.xml +2 -0
- data/features/match.feature +6 -6
- data/lib/recog/db.rb +33 -11
- data/lib/recog/db_manager.rb +6 -2
- data/lib/recog/fingerprint.rb +39 -2
- data/lib/recog/nizer.rb +93 -20
- data/lib/recog/version.rb +1 -1
- data/spec/lib/fingerprint_self_test_spec.rb +7 -0
- data/spec/lib/recog/nizer_spec.rb +165 -3
- data/xml/apache_os.xml +1 -1
- data/xml/architecture.xml +1 -1
- data/xml/fingerprints.xsd +91 -0
- data/xml/ftp_banners.xml +456 -74
- data/xml/h323_callresp.xml +1 -1
- data/xml/hp_pjl_id.xml +4 -1
- data/xml/http_cookies.xml +1 -1
- data/xml/http_servers.xml +1 -1
- data/xml/http_wwwauth.xml +1 -1
- data/xml/imap_banners.xml +1 -1
- data/xml/ldap_searchresult.xml +1 -1
- data/xml/mdns_device-info_txt.xml +1 -1
- data/xml/mdns_workstation_txt.xml +1 -1
- data/xml/mysql_banners.xml +1 -1
- data/xml/mysql_error.xml +1 -1
- data/xml/nntp_banners.xml +1 -1
- data/xml/ntp_banners.xml +1 -1
- data/xml/operating_system.xml +1 -1
- data/xml/pop_banners.xml +1 -1
- data/xml/rsh_resp.xml +1 -1
- data/xml/sip_banners.xml +1 -1
- data/xml/sip_user_agents.xml +1 -1
- data/xml/smb_native_lm.xml +1 -1
- data/xml/smb_native_os.xml +1 -1
- data/xml/smtp_banners.xml +5 -1
- data/xml/smtp_debug.xml +4 -1
- data/xml/smtp_ehlo.xml +4 -1
- data/xml/smtp_expn.xml +4 -1
- data/xml/smtp_help.xml +4 -1
- data/xml/smtp_mailfrom.xml +1 -1
- data/xml/smtp_noop.xml +4 -1
- data/xml/smtp_quit.xml +4 -1
- data/xml/smtp_rcptto.xml +1 -1
- data/xml/smtp_rset.xml +4 -1
- data/xml/smtp_turn.xml +4 -1
- data/xml/smtp_vrfy.xml +4 -1
- data/xml/snmp_sysdescr.xml +1 -1
- data/xml/snmp_sysobjid.xml +1 -1
- data/xml/ssh_banners.xml +1 -1
- data/xml/upnp_banners.xml +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11aff7826c38e379d719a70b39406f4d861f8d3f
|
4
|
+
data.tar.gz: 98f54e902871cb73ddcffbb7e0f72be5b21bf143
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 398dc88dfda3030d5d0e8da3c0774fc95ec99c2506f9f15ba528ef5ec930591f02f43d313ec5ed06cb4b7bb28798ef71bf53a539a6b50b8158a7f0b69877e0ed
|
7
|
+
data.tar.gz: 19bf9ddccb4031ade9bf5d94feb4516df400816eaf81a894b24716e06ef7e5d5a2400bd0424d55875b38451fcad83edb7f99f7b00ac1e38fa059447682f17d0a
|
data/README.md
CHANGED
@@ -75,7 +75,7 @@ Matches can be tested on the command-line in a similar fashion:
|
|
75
75
|
|
76
76
|
```
|
77
77
|
$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
|
78
|
-
MATCH: {"service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
78
|
+
MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
79
79
|
```
|
80
80
|
|
81
81
|
### Best Practices
|
@@ -1,5 +1,5 @@
|
|
1
1
|
<?xml version="1.0"?>
|
2
|
-
<fingerprints>
|
2
|
+
<fingerprints protocol="ftp" database_type="service">
|
3
3
|
<fingerprint pattern="^-{10} Welcome to Pure-FTPd (.*)-{10}$">
|
4
4
|
<example>---------- Welcome to Pure-FTPd ----------</example>
|
5
5
|
<description>Pure-FTPd
|
@@ -8,6 +8,7 @@
|
|
8
8
|
<param pos="1" name="pureftpd.config"/>
|
9
9
|
<param pos="0" name="service.family" value="Pure-FTPd"/>
|
10
10
|
<param pos="0" name="service.product" value="Pure-FTPd"/>
|
11
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
11
12
|
</fingerprint>
|
12
13
|
<fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
|
13
14
|
<description>SunOS/Solaris</description>
|
@@ -16,10 +16,12 @@
|
|
16
16
|
<param pos="1" name="pureftpd.config"/>
|
17
17
|
<param pos="0" name="service.family" value="Pure-FTPd"/>
|
18
18
|
<param pos="0" name="service.product" value="Pure-FTPd"/>
|
19
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
19
20
|
</fingerprint>
|
20
21
|
<fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
|
21
22
|
<description>SunOS/Solaris</description>
|
22
23
|
<example>example.com FTP server (SunOS 5.7) ready.</example>
|
24
|
+
<param pos="0" name="service.protocol" value="ftp"/>
|
23
25
|
<param pos="0" name="os.vendor" value="Sun"/>
|
24
26
|
<param pos="0" name="os.family" value="Solaris"/>
|
25
27
|
<param pos="0" name="os.product" value="Solaris"/>
|
data/features/match.feature
CHANGED
@@ -3,8 +3,8 @@ Feature: Match
|
|
3
3
|
When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
|
4
4
|
Then it should pass with:
|
5
5
|
"""
|
6
|
-
MATCH: {"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
7
|
-
MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
6
|
+
MATCH: {"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
7
|
+
MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
8
8
|
"""
|
9
9
|
|
10
10
|
Scenario: Fails at finding matches
|
@@ -19,14 +19,14 @@ Feature: Match
|
|
19
19
|
When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
|
20
20
|
Then it should pass with:
|
21
21
|
"""
|
22
|
-
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"},{"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
23
|
-
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
22
|
+
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"},{"matched"=>"Pure-FTPd Config data can be zero or more of: [privsep] [TLS]", "pureftpd.config"=>"[privsep] [TLS] ", "service.family"=>"Pure-FTPd", "service.product"=>"Pure-FTPd", "service.protocol"=>"ftp", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
23
|
+
MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
24
24
|
"""
|
25
25
|
|
26
26
|
Scenario: Finds first matches using no-multi-match flag
|
27
27
|
When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
|
28
28
|
Then it should pass with:
|
29
29
|
"""
|
30
|
-
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
31
|
-
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
30
|
+
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"---------- Welcome to Pure-FTPd [privsep] [TLS] ----------"}
|
31
|
+
MATCH: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
|
32
32
|
"""
|
data/lib/recog/db.rb
CHANGED
@@ -13,13 +13,34 @@ class DB
|
|
13
13
|
# against strings that make sense for the {#match_key}
|
14
14
|
attr_reader :fingerprints
|
15
15
|
|
16
|
-
# @return [String] Taken from the `fingerprints/matches`
|
16
|
+
# @return [String] Taken from the `fingerprints/matches` attribute, or
|
17
17
|
# defaults to the basename of {#path} without the `.xml` extension.
|
18
18
|
attr_reader :match_key
|
19
19
|
|
20
|
+
# @return [String] Taken from the `fingerprints/protocol` attribute, or
|
21
|
+
# defaults to an empty string
|
22
|
+
attr_reader :protocol
|
23
|
+
|
24
|
+
# @return [String] Taken from the `fingerprints/database_type` attribute
|
25
|
+
# defaults to an empty string
|
26
|
+
attr_reader :database_type
|
27
|
+
|
28
|
+
# @return [Float] Taken from the `fingerprints/preference` attribute,
|
29
|
+
# defaults to 0.10. Used when ordering databases, highest numbers
|
30
|
+
# are given priority and are processed first.
|
31
|
+
attr_reader :preference
|
32
|
+
|
33
|
+
# Default Fingerprint database preference when it isn't specified in file
|
34
|
+
# Do not use a value below 0.10 so as to allow users to specify lower
|
35
|
+
# values in their own custom XML that will always run last.
|
36
|
+
DEFAULT_FP_PREFERENCE = 0.10
|
37
|
+
|
20
38
|
# @param path [String]
|
21
39
|
def initialize(path)
|
22
40
|
@match_key = nil
|
41
|
+
@protocol = ''
|
42
|
+
@database_type = ''
|
43
|
+
@preference = DEFAULT_FP_PREFERENCE.to_f
|
23
44
|
@path = path
|
24
45
|
@fingerprints = []
|
25
46
|
|
@@ -30,24 +51,25 @@ class DB
|
|
30
51
|
def parse_fingerprints
|
31
52
|
xml = nil
|
32
53
|
|
33
|
-
File.open(self.path,
|
54
|
+
File.open(self.path, 'rb') do |fd|
|
34
55
|
xml = Nokogiri::XML(fd.read(fd.stat.size))
|
35
56
|
end
|
36
57
|
|
37
58
|
raise "#{self.path} is invalid XML: #{xml.errors.join(',')}" unless xml.errors.empty?
|
38
59
|
|
39
|
-
xml.xpath(
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
60
|
+
xml.xpath('/fingerprints').each do |fbase|
|
61
|
+
|
62
|
+
@match_key = fbase['matches'].to_s if fbase['matches']
|
63
|
+
@protocol = fbase['protocol'].to_s if fbase['protocol']
|
64
|
+
@database_type = fbase['database_type'].to_s if fbase['database_type']
|
65
|
+
@preference = fbase['preference'].to_f if fbase['preference']
|
44
66
|
|
45
|
-
unless @match_key
|
46
|
-
@match_key = File.basename(self.path).sub(/\.xml$/, '')
|
47
67
|
end
|
48
68
|
|
49
|
-
|
50
|
-
|
69
|
+
@match_key = File.basename(self.path).sub(/\.xml$/, '') unless @match_key
|
70
|
+
|
71
|
+
xml.xpath('/fingerprints/fingerprint').each do |fprint|
|
72
|
+
@fingerprints << Fingerprint.new(fprint, @match_key, @protocol)
|
51
73
|
end
|
52
74
|
|
53
75
|
xml = nil
|
data/lib/recog/db_manager.rb
CHANGED
@@ -13,8 +13,12 @@ class DBManager
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def load_databases
|
16
|
-
|
17
|
-
self.
|
16
|
+
if File.directory?(self.path)
|
17
|
+
Dir[self.path + "/*.xml"].each do |dbxml|
|
18
|
+
self.databases << DB.new(dbxml)
|
19
|
+
end
|
20
|
+
else
|
21
|
+
self.databases << DB.new(self.path)
|
18
22
|
end
|
19
23
|
end
|
20
24
|
|
data/lib/recog/fingerprint.rb
CHANGED
@@ -27,23 +27,49 @@ class Fingerprint
|
|
27
27
|
attr_reader :tests
|
28
28
|
|
29
29
|
# @param xml [Nokogiri::XML::Element]
|
30
|
-
|
30
|
+
# @param match_key [String] See Recog::DB
|
31
|
+
# @param protocol [String] Protocol such as ftp, mssql, http, etc.
|
32
|
+
def initialize(xml, match_key=nil, protocol=nil)
|
33
|
+
@match_key = match_key
|
34
|
+
@protocol = protocol
|
31
35
|
@name = parse_description(xml)
|
32
36
|
@regex = create_regexp(xml)
|
33
37
|
@params = {}
|
34
38
|
@tests = []
|
35
39
|
|
40
|
+
@protocol.downcase! if @protocol
|
36
41
|
parse_examples(xml)
|
37
42
|
parse_params(xml)
|
38
43
|
end
|
39
44
|
|
45
|
+
def output_diag_data(message, data, exception)
|
46
|
+
STDERR.puts message
|
47
|
+
STDERR.puts exception.inspect
|
48
|
+
STDERR.puts "Length: #{data.length}"
|
49
|
+
STDERR.puts "Encoding: #{data.encoding}"
|
50
|
+
STDERR.puts "Problematic data:\n#{data}"
|
51
|
+
STDERR.puts "Raw bytes:\n#{data.pretty_inspect}\n"
|
52
|
+
end
|
53
|
+
|
40
54
|
# Attempt to match the given string.
|
41
55
|
#
|
42
56
|
# @param match_string [String]
|
43
57
|
# @return [Hash,nil] Keys will be host, service, and os attributes
|
44
58
|
def match(match_string)
|
45
59
|
# match_string.force_encoding('BINARY') if match_string
|
46
|
-
|
60
|
+
begin
|
61
|
+
match_data = @regex.match(match_string)
|
62
|
+
rescue Encoding::CompatibilityError => e
|
63
|
+
begin
|
64
|
+
# Replace invalid UTF-8 characters with spaces, just as DAP does.
|
65
|
+
encoded_str = match_string.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => '')
|
66
|
+
match_data = @regex.match(encoded_str)
|
67
|
+
rescue Exception => e
|
68
|
+
output_diag_data('Exception while re-encoding match_string to UTF-8', match_string, e)
|
69
|
+
end
|
70
|
+
rescue Exception => e
|
71
|
+
output_diag_data('Exception while running regex against match_string', match_string, e)
|
72
|
+
end
|
47
73
|
return if match_data.nil?
|
48
74
|
|
49
75
|
result = { 'matched' => @name }
|
@@ -58,6 +84,17 @@ class Fingerprint
|
|
58
84
|
result[k] = match_data[ pos ]
|
59
85
|
end
|
60
86
|
end
|
87
|
+
|
88
|
+
# Use the protocol specified in the XML database if there isn't one
|
89
|
+
# provided as part of this fingerprint.
|
90
|
+
if @protocol
|
91
|
+
unless result['service.protocol']
|
92
|
+
result['service.protocol'] = @protocol
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
result['fingerprint_db'] = @match_key if @match_key
|
97
|
+
|
61
98
|
return result
|
62
99
|
end
|
63
100
|
|
data/lib/recog/nizer.rb
CHANGED
@@ -20,43 +20,116 @@ class Nizer
|
|
20
20
|
}
|
21
21
|
|
22
22
|
@@db_manager = nil
|
23
|
+
@@db_sorted = false
|
24
|
+
|
25
|
+
|
26
|
+
#
|
27
|
+
# Load fingerprints from a specific file or directory
|
28
|
+
# This will not preserve any fingerprints that have already been loaded
|
29
|
+
# @param path [String] Path to file or directory of XML fingerprints
|
30
|
+
def self.load_db(path = nil)
|
31
|
+
if path
|
32
|
+
@@db_manager = Recog::DBManager.new(path)
|
33
|
+
else
|
34
|
+
@@db_manager = Recog::DBManager.new
|
35
|
+
end
|
36
|
+
|
37
|
+
# Sort the databases, no behavior or result change for those calling
|
38
|
+
# Nizer.match or Nizer.multi_match as they have a single DB
|
39
|
+
@@db_manager.databases.sort! { |a, b| b.preference <=> a.preference }
|
40
|
+
@@db_sorted = true
|
41
|
+
end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Destroy the current DBManager object
|
45
|
+
def self.unload_db
|
46
|
+
@@db_manager = nil
|
47
|
+
@@db_sorted = false
|
48
|
+
end
|
23
49
|
|
50
|
+
#
|
51
|
+
# Display the fingerprint databases in the order in which they will be used
|
52
|
+
# to match banners. This is useful for fingerprint tuning and debugging.
|
53
|
+
def self.display_db_order
|
54
|
+
self.load_db unless @@db_manager
|
55
|
+
|
56
|
+
puts format('%s %-22s %-8s %s', 'Preference', 'Database', 'Type', 'Protocol')
|
57
|
+
@@db_manager.databases.each do |db|
|
58
|
+
puts format('%10.3f %-22s %-8s %s', db.preference, db.match_key,
|
59
|
+
db.database_type, db.protocol)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
#
|
64
|
+
# 2016.11 - Rewritten to be wrapper around #match_db_all, functionality
|
65
|
+
# and results must remain unchanged.
|
24
66
|
#
|
25
67
|
# Locate a database that corresponds with the `match_key` and attempt to
|
26
|
-
# find a matching {Fingerprint fingerprint}, stopping at the first hit.
|
27
|
-
# when no matching database or fingerprint is found.
|
68
|
+
# find a matching {Fingerprint fingerprint}, stopping at the first hit.
|
69
|
+
# Returns `nil` when no matching database or fingerprint is found.
|
28
70
|
#
|
29
71
|
# @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
|
30
|
-
# @
|
72
|
+
# @param match_string [String] String to match
|
73
|
+
# @return (see Fingerprint#match) or nil
|
31
74
|
def self.match(match_key, match_string)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
db.fingerprints.each do |fprint|
|
37
|
-
m = fprint.match(match_string)
|
38
|
-
return m if m
|
39
|
-
end
|
40
|
-
end
|
41
|
-
nil
|
75
|
+
filter = { match_key: match_key, multi_match: false }
|
76
|
+
matches = self.match_all_db(match_string, filter)
|
77
|
+
|
78
|
+
matches[0]
|
42
79
|
end
|
43
80
|
|
81
|
+
#
|
82
|
+
# @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
|
83
|
+
# @param match_string [String] String to match
|
84
|
+
# @return [Array] Array of Fingerprint#match or empty array
|
44
85
|
def self.multi_match(match_key, match_string)
|
45
|
-
|
46
|
-
|
86
|
+
filter = { match_key: match_key, multi_match: true }
|
87
|
+
self.match_all_db(match_string, filter)
|
88
|
+
end
|
47
89
|
|
48
|
-
|
90
|
+
#
|
91
|
+
# Search all fingerprint dbs and attempt to find matching
|
92
|
+
# {Fingerprint fingerprint}s. It will return the first match found
|
93
|
+
# unless the :multi_match option is used to request all matches.
|
94
|
+
# Returns an array of all matching fingerprints or an empty array.
|
95
|
+
#
|
96
|
+
# @param match_string [String] Service banner to match
|
97
|
+
# @param [Hash] filters This hash contains filters used to limit the
|
98
|
+
# results to just those from specific types of fingerprints.
|
99
|
+
# The values that these filters match come from the 'fingerprints' top
|
100
|
+
# level element in the fingerprint DB XML or, in the case of 'protocol',
|
101
|
+
# this value can be overridden at the individual fingerprint level by
|
102
|
+
# setting a value for 'service.protocol'
|
103
|
+
#
|
104
|
+
# With the exception of 'match_key', the filters below match the
|
105
|
+
# 'fingerprints' attributes with the same name.
|
106
|
+
# @option filters [String] :match_key Value from XML 'matches' or file name
|
107
|
+
# @option filters [String] :database_type fprint db type: service, util.os, etc.
|
108
|
+
# @option filters [String] :protocol Protocol (ftp, smtp, etc.)
|
109
|
+
# @option filters [Boolean] :multi_match Return all matches instead of first
|
110
|
+
# @return [Array] Array of Fingerprint#match or empty array
|
111
|
+
def self.match_all_db(match_string, filters = {})
|
112
|
+
match_string = match_string.to_s.unpack('C*').pack('C*')
|
113
|
+
matches = Array.new # array to hold all fingerprint matches
|
114
|
+
|
115
|
+
self.load_db unless @@db_manager
|
49
116
|
|
50
117
|
@@db_manager.databases.each do |db|
|
51
|
-
next
|
52
|
-
|
118
|
+
next if filters[:match_key] && !filters[:match_key].eql?(db.match_key)
|
119
|
+
next if filters[:database_type] && !filters[:database_type].eql?(db.database_type)
|
53
120
|
db.fingerprints.each do |fp|
|
54
121
|
m = fp.match(match_string)
|
55
|
-
|
122
|
+
if m
|
123
|
+
# Filter on protocol after match since each individual fp
|
124
|
+
# can contain its own 'protocol' value that overrides the
|
125
|
+
# one set at the DB level.
|
126
|
+
matches.push(m) unless filters[:protocol] && !filters[:protocol].eql?(m['service.protocol'])
|
127
|
+
return matches unless filters[:multi_match]
|
128
|
+
end
|
56
129
|
end
|
57
130
|
end
|
58
131
|
|
59
|
-
|
132
|
+
matches
|
60
133
|
end
|
61
134
|
|
62
135
|
#
|
data/lib/recog/version.rb
CHANGED
@@ -21,6 +21,13 @@ describe Recog::DB do
|
|
21
21
|
expect(db.match_key).not_to be_empty
|
22
22
|
end
|
23
23
|
|
24
|
+
it "has valid 'preference' value" do
|
25
|
+
# Reserve values below 0.10 and above 0.90 for users
|
26
|
+
# See xml/fingerprints.xsd
|
27
|
+
expect(db.preference.class).to be ::Float
|
28
|
+
expect(db.preference).to be_between(0.10, 0.90)
|
29
|
+
end
|
30
|
+
|
24
31
|
db.fingerprints.each_index do |i|
|
25
32
|
fp = db.fingerprints[i]
|
26
33
|
|
@@ -1,6 +1,12 @@
|
|
1
1
|
require 'recog'
|
2
2
|
require 'yaml'
|
3
3
|
|
4
|
+
|
5
|
+
VALID_FILTER = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'util.os'}
|
6
|
+
NOMATCH_MATCH_KEY = {match_key: 'no_such_987', protocol: 'smb', database_type: 'util.os'}
|
7
|
+
NOMATCH_PROTO = {match_key: 'smb.native_os', protocol: 'no_such_987', database_type: 'util.os'}
|
8
|
+
NOMATCH_TYPE = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'no_such_987'}
|
9
|
+
|
4
10
|
describe Recog::Nizer do
|
5
11
|
subject { described_class }
|
6
12
|
|
@@ -24,6 +30,15 @@ describe Recog::Nizer do
|
|
24
30
|
end
|
25
31
|
end
|
26
32
|
|
33
|
+
let(:nomatch_result) { subject.match('smb.native_os', 'no_such_987_76tgklh') }
|
34
|
+
it "returns a nil when data cannot be matched" do
|
35
|
+
expect(nomatch_result).to be_nil
|
36
|
+
end
|
37
|
+
|
38
|
+
let(:invalid_db_result) { subject.match('no_such_987', data) }
|
39
|
+
it "returns a nil when match_key search doesn't match" do
|
40
|
+
expect(invalid_db_result).to be_nil
|
41
|
+
end
|
27
42
|
end
|
28
43
|
end
|
29
44
|
|
@@ -36,6 +51,83 @@ describe Recog::Nizer do
|
|
36
51
|
end
|
37
52
|
end
|
38
53
|
|
54
|
+
describe ".match_all_db" do
|
55
|
+
File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
|
56
|
+
data = line.strip
|
57
|
+
context "with smb_native_os:#{data}" do
|
58
|
+
let(:match_all_result) { subject.match_all_db(data, VALID_FILTER) }
|
59
|
+
|
60
|
+
it "returns an array" do
|
61
|
+
expect(match_all_result.class).to eq(::Array)
|
62
|
+
end
|
63
|
+
|
64
|
+
it "returns a successful match" do
|
65
|
+
expect(match_all_result[0]['matched']).to match(/^[A-Z]/)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "correctly matches service or os" do
|
69
|
+
if data =~ /^Windows/
|
70
|
+
expect(match_all_result[0]['os.product']).to match(/^Windows/)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
it "correctly matches protocol" do
|
75
|
+
expect(match_all_result[0]['service.protocol']).to eq('smb')
|
76
|
+
end
|
77
|
+
|
78
|
+
let(:no_filter_result) { subject.match_all_db(data) }
|
79
|
+
it "returns an array when searching without a filter" do
|
80
|
+
expect(no_filter_result.class).to eq(::Array)
|
81
|
+
end
|
82
|
+
|
83
|
+
it "returns a successful match when searching without a filter" do
|
84
|
+
expect(no_filter_result[0]['matched']).to match(/^[A-Z]/)
|
85
|
+
end
|
86
|
+
|
87
|
+
it "correctly matches service or os when searching without a filter" do
|
88
|
+
if data =~ /^Windows/
|
89
|
+
expect(no_filter_result[0]['os.product']).to match(/^Windows/)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
let(:nomatch_db_result) { subject.match_all_db(data, NOMATCH_MATCH_KEY) }
|
94
|
+
it "returns an array when match_key search doesn't match" do
|
95
|
+
expect(nomatch_db_result.class).to eq(::Array)
|
96
|
+
end
|
97
|
+
it "returns an empty array when match_key search doesn't match" do
|
98
|
+
expect(nomatch_db_result).to be_empty
|
99
|
+
end
|
100
|
+
|
101
|
+
let(:nomatch_proto_result) { subject.match_all_db(data, NOMATCH_PROTO) }
|
102
|
+
it "returns an array when protocol search doesn't match" do
|
103
|
+
expect(nomatch_proto_result.class).to eq(::Array)
|
104
|
+
end
|
105
|
+
it "returns an empty array when protocol search doesn't match" do
|
106
|
+
expect(nomatch_proto_result).to be_empty
|
107
|
+
end
|
108
|
+
|
109
|
+
let(:nomatch_type_result) { subject.match_all_db(data, NOMATCH_TYPE) }
|
110
|
+
it "returns an array when database_type search doesn't match" do
|
111
|
+
expect(nomatch_type_result.class).to eq(::Array)
|
112
|
+
end
|
113
|
+
it "returns an empty array when database_type search doesn't match" do
|
114
|
+
expect(nomatch_proto_result).to be_empty
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
line = 'non-existent'
|
120
|
+
context "with non-existent match" do
|
121
|
+
let(:match_result) {subject.match_all_db(line) }
|
122
|
+
it "returns an array" do
|
123
|
+
expect(match_result.class).to eq(::Array)
|
124
|
+
end
|
125
|
+
it "returns an empty array" do
|
126
|
+
expect(match_result).to be_empty
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
39
131
|
describe ".multi_match" do
|
40
132
|
File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
|
41
133
|
data = line.strip
|
@@ -58,8 +150,46 @@ describe Recog::Nizer do
|
|
58
150
|
end
|
59
151
|
end
|
60
152
|
end
|
153
|
+
|
154
|
+
let(:invalid_db_result) { subject.multi_match('no_such_987', data) }
|
155
|
+
it "returns an array when passed an invalid match_key" do
|
156
|
+
expect(invalid_db_result.class).to eq(::Array)
|
157
|
+
end
|
158
|
+
|
159
|
+
it "returns an empty array when passed an invalid match_key" do
|
160
|
+
expect(invalid_db_result).to be_empty
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
end
|
165
|
+
|
166
|
+
data = 'Windows Server 2012 R2 Standard 9600'
|
167
|
+
context "with {data}" do
|
168
|
+
let(:match_results) {subject.multi_match('smb.native_os', data) }
|
169
|
+
|
170
|
+
it "returns an array" do
|
171
|
+
expect(match_results.class).to eq(::Array)
|
172
|
+
end
|
173
|
+
|
174
|
+
it "returns at least two successful matches" do
|
175
|
+
expect(match_results.size).to be > 1
|
176
|
+
end
|
177
|
+
|
178
|
+
it "correctly matches os.product for all matches" do
|
179
|
+
match_results do |mr|
|
180
|
+
if data =~ /^Windows/
|
181
|
+
expect(mr['os.product']).to match(/^Windows/)
|
182
|
+
end
|
183
|
+
end
|
61
184
|
end
|
62
185
|
|
186
|
+
it "correctly matches protocol for all matches" do
|
187
|
+
match_results do |mr|
|
188
|
+
if data =~ /^Windows/
|
189
|
+
expect(mr['service.protocol']).to eq('smb')
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
63
193
|
end
|
64
194
|
|
65
195
|
line = 'non-existent'
|
@@ -77,7 +207,6 @@ describe Recog::Nizer do
|
|
77
207
|
end
|
78
208
|
|
79
209
|
describe ".best_os_match" do
|
80
|
-
|
81
210
|
# Demonstrates how this method picks up additional attributes from other members of the winning
|
82
211
|
# os.product match group and applies them to the result.
|
83
212
|
matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_1.yml'))))
|
@@ -134,8 +263,7 @@ describe Recog::Nizer do
|
|
134
263
|
|
135
264
|
end
|
136
265
|
|
137
|
-
|
138
|
-
|
266
|
+
describe ".best_service_match" do
|
139
267
|
# Demonstrates how this method picks up additional attributes from other members of the winning
|
140
268
|
# service.product match group and applies them to the result.
|
141
269
|
matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_service_match_1.yml'))))
|
@@ -165,4 +293,38 @@ describe Recog::Nizer do
|
|
165
293
|
|
166
294
|
end
|
167
295
|
|
296
|
+
|
297
|
+
describe '.load_db' do
|
298
|
+
file_path = File.expand_path(File.join('spec', 'data', 'test_fingerprints.xml'))
|
299
|
+
context "with #{file_path}" do
|
300
|
+
let(:fp_db) { subject.load_db(file_path) }
|
301
|
+
it "loads without error" do
|
302
|
+
expect(fp_db).to be true
|
303
|
+
subject.unload_db()
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
context "with no path specified" do
|
308
|
+
let(:fp_db) { subject.load_db }
|
309
|
+
it "loads without error" do
|
310
|
+
expect(fp_db).to be true
|
311
|
+
subject.unload_db()
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
context "with empty file path" do
|
316
|
+
it "raises an error" do
|
317
|
+
expect { subject.load_db('') }.to raise_error(Errno::ENOENT)
|
318
|
+
subject.unload_db()
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
context "with invalid file path" do
|
323
|
+
it "raises an error" do
|
324
|
+
expect { subject.load_db('no_such_987_file_path') }.to raise_error(Errno::ENOENT)
|
325
|
+
subject.unload_db()
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
168
330
|
end
|