recog 3.1.1 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/Gemfile +6 -0
- data/Rakefile +7 -5
- data/lib/recog/db.rb +67 -68
- data/lib/recog/db_manager.rb +22 -21
- data/lib/recog/fingerprint/regexp_factory.rb +10 -13
- data/lib/recog/fingerprint/test.rb +9 -8
- data/lib/recog/fingerprint.rb +252 -262
- data/lib/recog/fingerprint_parse_error.rb +3 -1
- data/lib/recog/formatter.rb +41 -39
- data/lib/recog/match_reporter.rb +82 -83
- data/lib/recog/matcher.rb +37 -40
- data/lib/recog/matcher_factory.rb +7 -6
- data/lib/recog/nizer.rb +218 -224
- data/lib/recog/verifier.rb +30 -28
- data/lib/recog/verify_reporter.rb +69 -73
- data/lib/recog/version.rb +3 -1
- data/lib/recog.rb +2 -0
- data/recog/bin/recog_match +21 -20
- data/recog/xml/apache_modules.xml +2 -0
- data/recog/xml/dhcp_vendor_class.xml +1 -1
- data/recog/xml/favicons.xml +133 -1
- data/recog/xml/ftp_banners.xml +1 -1
- data/recog/xml/html_title.xml +140 -1
- data/recog/xml/http_cookies.xml +20 -2
- data/recog/xml/http_servers.xml +38 -17
- data/recog/xml/http_wwwauth.xml +17 -4
- data/recog/xml/mdns_device-info_txt.xml +49 -15
- data/recog/xml/sip_banners.xml +0 -2
- data/recog/xml/sip_user_agents.xml +1 -1
- data/recog/xml/snmp_sysdescr.xml +1 -2
- data/recog/xml/ssh_banners.xml +8 -0
- data/recog/xml/telnet_banners.xml +3 -2
- data/recog/xml/tls_jarm.xml +1 -1
- data/recog/xml/x11_banners.xml +1 -0
- data/recog/xml/x509_issuers.xml +1 -1
- data/recog/xml/x509_subjects.xml +0 -1
- data/recog.gemspec +14 -13
- data/spec/lib/recog/db_spec.rb +37 -36
- data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +19 -20
- data/spec/lib/recog/fingerprint_spec.rb +44 -42
- data/spec/lib/recog/formatter_spec.rb +20 -18
- data/spec/lib/recog/match_reporter_spec.rb +35 -30
- data/spec/lib/recog/nizer_spec.rb +85 -101
- data/spec/lib/recog/verify_reporter_spec.rb +45 -44
- data/spec/spec_helper.rb +2 -1
- data.tar.gz.sig +1 -3
- metadata +3 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 06e7e85a8a5531bf315f482708ed3ada966be06efea9d09d85add533d098c2f9
|
4
|
+
data.tar.gz: e683dd6e7c529d95eef78cf1ebf32a19594449a2105593234860619393643aa0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 69bc3fe5c446de07ee4234f668293839c902eb7df8dbd89e5e5b410a8d43bcddecd4cda0d0acb513c3f262d6aa5496e3ade0a2ad4df02b724a6b5b21edc37f9c
|
7
|
+
data.tar.gz: 2fb2209fcc598a8a592e4f81ced771dc5520e4cb46f06277e8e9a47e3827f7d0f4d0348a6c41bb759eb538943e06694c3b1f34d04fe09c235ac375110c2cedb1
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1,15 +1,17 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bundler/gem_tasks'
|
2
4
|
|
3
5
|
require 'rspec/core/rake_task'
|
4
6
|
RSpec::Core::RakeTask.new do |t|
|
5
|
-
|
7
|
+
t.pattern = ['spec/**/*_spec.rb', 'recog/spec/**/*_spec.rb']
|
6
8
|
end
|
7
9
|
|
8
10
|
require 'yard'
|
9
11
|
require 'yard/rake/yardoc_task'
|
10
12
|
YARD::Rake::YardocTask.new do |t|
|
11
|
-
|
13
|
+
t.files = ['lib/**/*.rb', '-', 'README.md']
|
12
14
|
end
|
13
15
|
|
14
|
-
task :
|
15
|
-
task :
|
16
|
+
task default: %i[tests yard]
|
17
|
+
task tests: [:spec]
|
data/lib/recog/db.rb
CHANGED
@@ -1,79 +1,78 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
# A collection of {Fingerprint fingerprints} for matching against a particular
|
4
|
-
# kind of fingerprintable data, e.g. an HTTP `Server` header
|
5
|
-
class DB
|
6
|
-
require 'nokogiri'
|
7
|
-
require 'recog/fingerprint'
|
8
|
-
|
9
|
-
# @return [String]
|
10
|
-
attr_reader :path
|
11
|
-
|
12
|
-
# @return [Array<Fingerprint>] {Fingerprint} objects that can be matched
|
13
|
-
# against strings that make sense for the {#match_key}
|
14
|
-
attr_reader :fingerprints
|
15
|
-
|
16
|
-
# @return [String] Taken from the `fingerprints/matches` attribute, or
|
17
|
-
# defaults to the basename of {#path} without the `.xml` extension.
|
18
|
-
attr_reader :match_key
|
19
|
-
|
20
|
-
# @return [String] Taken from the `fingerprints/protocol` attribute, or
|
21
|
-
# defaults to an empty string
|
22
|
-
attr_reader :protocol
|
23
|
-
|
24
|
-
# @return [String] Taken from the `fingerprints/database_type` attribute
|
25
|
-
# defaults to an empty string
|
26
|
-
attr_reader :database_type
|
27
|
-
|
28
|
-
# @return [Float] Taken from the `fingerprints/preference` attribute,
|
29
|
-
# defaults to 0.10. Used when ordering databases, highest numbers
|
30
|
-
# are given priority and are processed first.
|
31
|
-
attr_reader :preference
|
32
|
-
|
33
|
-
# Default Fingerprint database preference when it isn't specified in file
|
34
|
-
# Do not use a value below 0.10 so as to allow users to specify lower
|
35
|
-
# values in their own custom XML that will always run last.
|
36
|
-
DEFAULT_FP_PREFERENCE = 0.10
|
37
|
-
|
38
|
-
# @param path [String]
|
39
|
-
def initialize(path)
|
40
|
-
@match_key = nil
|
41
|
-
@protocol = ''
|
42
|
-
@database_type = ''
|
43
|
-
@preference = DEFAULT_FP_PREFERENCE.to_f
|
44
|
-
@path = path
|
45
|
-
@fingerprints = []
|
46
|
-
|
47
|
-
parse_fingerprints
|
48
|
-
end
|
1
|
+
# frozen_string_literal: true
|
49
2
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
3
|
+
module Recog
|
4
|
+
# A collection of {Fingerprint fingerprints} for matching against a particular
|
5
|
+
# kind of fingerprintable data, e.g. an HTTP `Server` header
|
6
|
+
class DB
|
7
|
+
require 'nokogiri'
|
8
|
+
require 'recog/fingerprint'
|
9
|
+
|
10
|
+
# @return [String]
|
11
|
+
attr_reader :path
|
12
|
+
|
13
|
+
# @return [Array<Fingerprint>] {Fingerprint} objects that can be matched
|
14
|
+
# against strings that make sense for the {#match_key}
|
15
|
+
attr_reader :fingerprints
|
16
|
+
|
17
|
+
# @return [String] Taken from the `fingerprints/matches` attribute, or
|
18
|
+
# defaults to the basename of {#path} without the `.xml` extension.
|
19
|
+
attr_reader :match_key
|
20
|
+
|
21
|
+
# @return [String] Taken from the `fingerprints/protocol` attribute, or
|
22
|
+
# defaults to an empty string
|
23
|
+
attr_reader :protocol
|
24
|
+
|
25
|
+
# @return [String] Taken from the `fingerprints/database_type` attribute
|
26
|
+
# defaults to an empty string
|
27
|
+
attr_reader :database_type
|
28
|
+
|
29
|
+
# @return [Float] Taken from the `fingerprints/preference` attribute,
|
30
|
+
# defaults to 0.10. Used when ordering databases, highest numbers
|
31
|
+
# are given priority and are processed first.
|
32
|
+
attr_reader :preference
|
33
|
+
|
34
|
+
# Default Fingerprint database preference when it isn't specified in file
|
35
|
+
# Do not use a value below 0.10 so as to allow users to specify lower
|
36
|
+
# values in their own custom XML that will always run last.
|
37
|
+
DEFAULT_FP_PREFERENCE = 0.10
|
38
|
+
|
39
|
+
# @param path [String]
|
40
|
+
def initialize(path)
|
41
|
+
@match_key = nil
|
42
|
+
@protocol = ''
|
43
|
+
@database_type = ''
|
44
|
+
@preference = DEFAULT_FP_PREFERENCE.to_f
|
45
|
+
@path = path
|
46
|
+
@fingerprints = []
|
47
|
+
|
48
|
+
parse_fingerprints
|
56
49
|
end
|
57
50
|
|
58
|
-
|
51
|
+
# @return [void]
|
52
|
+
def parse_fingerprints
|
53
|
+
xml = nil
|
59
54
|
|
60
|
-
|
55
|
+
File.open(path, 'rb') do |fd|
|
56
|
+
xml = Nokogiri::XML(fd.read(fd.stat.size))
|
57
|
+
end
|
61
58
|
|
62
|
-
|
63
|
-
@protocol = fbase['protocol'].to_s if fbase['protocol']
|
64
|
-
@database_type = fbase['database_type'].to_s if fbase['database_type']
|
65
|
-
@preference = fbase['preference'].to_f if fbase['preference']
|
59
|
+
raise "#{path} is invalid XML: #{xml.errors.join(',')}" unless xml.errors.empty?
|
66
60
|
|
67
|
-
|
61
|
+
xml.xpath('/fingerprints').each do |fbase|
|
62
|
+
@match_key = fbase['matches'].to_s if fbase['matches']
|
63
|
+
@protocol = fbase['protocol'].to_s if fbase['protocol']
|
64
|
+
@database_type = fbase['database_type'].to_s if fbase['database_type']
|
65
|
+
@preference = fbase['preference'].to_f if fbase['preference']
|
66
|
+
end
|
68
67
|
|
69
|
-
|
70
|
-
|
68
|
+
filepath = path.sub(/\.xml$/, '')
|
69
|
+
@match_key ||= File.basename(path).sub(/\.xml$/, '')
|
71
70
|
|
72
|
-
|
73
|
-
|
74
|
-
|
71
|
+
xml.xpath('/fingerprints/fingerprint').each do |fprint|
|
72
|
+
@fingerprints << Fingerprint.new(fprint, @match_key, @protocol, filepath)
|
73
|
+
end
|
75
74
|
|
76
|
-
|
75
|
+
xml = nil
|
76
|
+
end
|
77
77
|
end
|
78
78
|
end
|
79
|
-
end
|
data/lib/recog/db_manager.rb
CHANGED
@@ -1,31 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Recog
|
2
|
-
class DBManager
|
3
|
-
|
4
|
-
|
4
|
+
class DBManager
|
5
|
+
require 'nokogiri'
|
6
|
+
require 'recog/db'
|
5
7
|
|
6
|
-
|
8
|
+
attr_accessor :path, :databases
|
7
9
|
|
8
|
-
|
10
|
+
DefaultDatabasePath = File.expand_path(File.join(File.expand_path(__dir__), ['..', '..', 'recog', 'xml']))
|
9
11
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
def initialize(path = DefaultDatabasePath)
|
13
|
+
self.path = path
|
14
|
+
reload
|
15
|
+
end
|
14
16
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
def load_databases
|
18
|
+
if File.directory?(path)
|
19
|
+
Dir["#{path}/*.xml"].each do |dbxml|
|
20
|
+
databases << DB.new(dbxml)
|
21
|
+
end
|
22
|
+
else
|
23
|
+
databases << DB.new(path)
|
19
24
|
end
|
20
|
-
else
|
21
|
-
self.databases << DB.new(self.path)
|
22
25
|
end
|
23
|
-
end
|
24
26
|
|
25
|
-
|
26
|
-
|
27
|
-
|
27
|
+
def reload
|
28
|
+
self.databases = []
|
29
|
+
load_databases
|
30
|
+
end
|
28
31
|
end
|
29
|
-
|
30
|
-
end
|
31
32
|
end
|
@@ -1,14 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
module Recog
|
3
4
|
class Fingerprint
|
4
|
-
|
5
5
|
#
|
6
6
|
# @example
|
7
7
|
# r = RegexpFactory.build("^Apache[ -]Coyote/(\d\.\d)$", "REG_ICASE")
|
8
8
|
# r.match("Apache-Coyote/1.1")
|
9
9
|
#
|
10
10
|
module RegexpFactory
|
11
|
-
|
12
11
|
# Currently, only options relating to case insensitivity and
|
13
12
|
# multiline/newline are supported. Because Recog's data is used by tools
|
14
13
|
# written in different languages like Ruby and Java, we currently support
|
@@ -19,13 +18,13 @@ module Recog
|
|
19
18
|
# that use Recog data translate accordingly
|
20
19
|
FLAG_MAP = {
|
21
20
|
# multiline variations
|
22
|
-
'REG_DOT_NEWLINE'
|
21
|
+
'REG_DOT_NEWLINE' => Regexp::MULTILINE,
|
23
22
|
'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
|
24
|
-
'REG_MULTILINE'
|
23
|
+
'REG_MULTILINE' => Regexp::MULTILINE,
|
25
24
|
# case variations
|
26
|
-
'REG_ICASE'
|
27
|
-
'IGNORECASE'
|
28
|
-
}
|
25
|
+
'REG_ICASE' => Regexp::IGNORECASE,
|
26
|
+
'IGNORECASE' => Regexp::IGNORECASE
|
27
|
+
}.freeze
|
29
28
|
|
30
29
|
DEFAULT_FLAGS = 0
|
31
30
|
|
@@ -42,15 +41,13 @@ module Recog
|
|
42
41
|
# @param flags [Array<String>]
|
43
42
|
# @return [Fixnum] Flags for creating a regular expression object
|
44
43
|
def self.build_options(flags)
|
45
|
-
unsupported_flags = flags.
|
46
|
-
unless unsupported_flags.empty?
|
47
|
-
|
48
|
-
end
|
44
|
+
unsupported_flags = flags.reject { |flag| FLAG_MAP.key?(flag) }
|
45
|
+
raise "Unsupported regular expression flags found: #{unsupported_flags.join(',')}. Must be one of: #{FLAG_MAP.keys.join(',')}" unless unsupported_flags.empty?
|
46
|
+
|
49
47
|
flags.reduce(DEFAULT_FLAGS) do |sum, flag|
|
50
|
-
sum
|
48
|
+
sum | (FLAG_MAP[flag] || 0)
|
51
49
|
end
|
52
50
|
end
|
53
51
|
end
|
54
52
|
end
|
55
53
|
end
|
56
|
-
|
@@ -1,15 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
|
2
3
|
class Recog::Fingerprint::Test
|
3
|
-
attr_accessor :content
|
4
|
-
|
5
|
-
def initialize(content, attributes=[])
|
4
|
+
attr_accessor :content, :attributes
|
5
|
+
|
6
|
+
def initialize(content, attributes = [])
|
6
7
|
@attributes = attributes
|
7
8
|
|
8
|
-
if @attributes['_encoding'] && @attributes['_encoding'] == 'base64'
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
@content = if @attributes['_encoding'] && @attributes['_encoding'] == 'base64'
|
10
|
+
content.to_s.unpack1('m*')
|
11
|
+
else
|
12
|
+
content
|
13
|
+
end
|
13
14
|
end
|
14
15
|
|
15
16
|
def to_s
|