recog 0.02 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +6 -0
- data/.rspec +2 -1
- data/.travis.yml +5 -0
- data/.yardopts +1 -0
- data/Gemfile +3 -1
- data/README.md +12 -12
- data/Rakefile +22 -0
- data/bin/recog_verify.rb +1 -1
- data/features/match.feature +2 -2
- data/features/verify.feature +10 -7
- data/features/xml/no_tests.xml +0 -50
- data/features/xml/successful_tests.xml +7 -22
- data/features/xml/tests_with_failures.xml +10 -0
- data/features/xml/tests_with_warnings.xml +7 -0
- data/lib/recog/db.rb +26 -10
- data/lib/recog/db_manager.rb +1 -1
- data/lib/recog/fingerprint.rb +118 -34
- data/lib/recog/fingerprint/regexp_factory.rb +39 -0
- data/lib/recog/fingerprint/test.rb +13 -0
- data/lib/recog/matcher.rb +3 -3
- data/lib/recog/nizer.rb +16 -23
- data/lib/recog/verifier.rb +10 -25
- data/lib/recog/verifier_factory.rb +1 -1
- data/lib/recog/verify_reporter.rb +1 -1
- data/lib/recog/version.rb +1 -1
- data/recog.gemspec +12 -3
- data/spec/data/test_fingerprints.xml +12 -0
- data/spec/lib/fingerprint_self_test_spec.rb +8 -4
- data/spec/lib/{db_spec.rb → recog/db_spec.rb} +19 -7
- data/spec/lib/recog/fingerprint/regexp_factory.rb +61 -0
- data/spec/lib/recog/fingerprint_spec.rb +5 -0
- data/spec/lib/{formatter_spec.rb → recog/formatter_spec.rb} +1 -1
- data/spec/lib/{match_reporter_spec.rb → recog/match_reporter_spec.rb} +10 -9
- data/spec/lib/{nizer_spec.rb → recog/nizer_spec.rb} +5 -5
- data/spec/lib/{verify_reporter_spec.rb → recog/verify_reporter_spec.rb} +8 -7
- data/spec/spec_helper.rb +82 -0
- data/xml/apache_os.xml +48 -2
- data/xml/http_servers.xml +38 -6
- data/xml/ntp_banners.xml +4 -3
- data/xml/smb_native_os.xml +32 -32
- data/xml/smtp_expn.xml +1 -0
- data/xml/smtp_help.xml +2 -1
- data/xml/snmp_sysdescr.xml +164 -24
- data/xml/ssh_banners.xml +7 -3
- metadata +56 -8
- data/Gemfile.lock +0 -42
@@ -0,0 +1,39 @@
|
|
1
|
+
|
2
|
+
module Recog
|
3
|
+
class Fingerprint
|
4
|
+
|
5
|
+
#
|
6
|
+
# @example
|
7
|
+
# r = RegexpFactory.build("^Apache[ -]Coyote/(\d\.\d)$", "REG_ICASE")
|
8
|
+
# r.match("Apache-Coyote/1.1")
|
9
|
+
#
|
10
|
+
module RegexpFactory
|
11
|
+
|
12
|
+
# Map strings as used in Recog XML to Fixnum values used by Regexp
|
13
|
+
FLAG_MAP = {
|
14
|
+
'REG_DOT_NEWLINE' => Regexp::MULTILINE,
|
15
|
+
'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
|
16
|
+
'REG_ICASE' => Regexp::IGNORECASE,
|
17
|
+
}
|
18
|
+
|
19
|
+
# @return [Regexp]
|
20
|
+
def self.build(pattern, flags)
|
21
|
+
options = build_options(flags)
|
22
|
+
Regexp.new(pattern, options)
|
23
|
+
end
|
24
|
+
|
25
|
+
# Convert string flag names as used in Recog XML into a Fixnum suitable for
|
26
|
+
# passing as the `options` parameter to `Regexp.new`
|
27
|
+
#
|
28
|
+
# @see FLAG_MAP
|
29
|
+
# @param flags [Array<String>]
|
30
|
+
# @return [Fixnum] Flags for creating a regular expression object
|
31
|
+
def self.build_options(flags)
|
32
|
+
flags.reduce(Regexp::NOENCODING) do |sum, flag|
|
33
|
+
sum |= (FLAG_MAP[flag] || 0)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
data/lib/recog/matcher.rb
CHANGED
@@ -9,7 +9,7 @@ class Matcher
|
|
9
9
|
|
10
10
|
def match_banners(banners_file)
|
11
11
|
reporter.report do
|
12
|
-
|
12
|
+
|
13
13
|
fd = $stdin
|
14
14
|
file_source = false
|
15
15
|
|
@@ -22,7 +22,7 @@ class Matcher
|
|
22
22
|
reporter.increment_line_count
|
23
23
|
|
24
24
|
line = line.to_s.unpack("C*").pack("C*").strip.gsub(/\\[rn]/, '')
|
25
|
-
found = nil
|
25
|
+
found = nil
|
26
26
|
fingerprints.each do |fp|
|
27
27
|
m = line.match(fp.regex)
|
28
28
|
if m
|
@@ -53,7 +53,7 @@ class Matcher
|
|
53
53
|
end
|
54
54
|
|
55
55
|
fd.close if file_source
|
56
|
-
|
56
|
+
|
57
57
|
end
|
58
58
|
end
|
59
59
|
end
|
data/lib/recog/nizer.rb
CHANGED
@@ -22,27 +22,20 @@ class Nizer
|
|
22
22
|
@@db_manager = nil
|
23
23
|
|
24
24
|
#
|
25
|
-
# Locate a database that corresponds with the match_key and attempt to
|
26
|
-
# find a matching
|
27
|
-
#
|
25
|
+
# Locate a database that corresponds with the `match_key` and attempt to
|
26
|
+
# find a matching {Fingerprint fingerprint}, stopping at the first hit. Returns `nil`
|
27
|
+
# when no matching database or fingerprint is found.
|
28
28
|
#
|
29
|
+
# @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
|
30
|
+
# @return (see Fingerprint#match)
|
29
31
|
def self.match(match_key, match_string)
|
30
32
|
match_string = match_string.to_s.unpack("C*").pack("C*")
|
31
33
|
@@db_manager ||= Recog::DBManager.new
|
32
34
|
@@db_manager.databases.each do |db|
|
33
35
|
next unless db.match_key == match_key
|
34
36
|
db.fingerprints.each do |fprint|
|
35
|
-
m = fprint.
|
36
|
-
|
37
|
-
result = { 'matched' => fprint.name }
|
38
|
-
fprint.params.each_pair do |k,v|
|
39
|
-
if v[0] == 0
|
40
|
-
result[k] = v[1]
|
41
|
-
else
|
42
|
-
result[k] = m[ v[0] ]
|
43
|
-
end
|
44
|
-
end
|
45
|
-
return result
|
37
|
+
m = fprint.match(match_string)
|
38
|
+
return m if m
|
46
39
|
end
|
47
40
|
end
|
48
41
|
nil
|
@@ -69,7 +62,7 @@ class Nizer
|
|
69
62
|
(HOST_ATTRIBUTES & m.keys).each do |ha|
|
70
63
|
host_attrs[ha] ||= {}
|
71
64
|
host_attrs[ha][m[ha]] ||= 0
|
72
|
-
host_attrs[ha][m[ha]] += 1
|
65
|
+
host_attrs[ha][m[ha]] += 1
|
73
66
|
end
|
74
67
|
|
75
68
|
next unless m.has_key?('os.product')
|
@@ -85,7 +78,7 @@ class Nizer
|
|
85
78
|
# Select the best host attribute value by highest frequency
|
86
79
|
#
|
87
80
|
host_attrs.keys.each do |hk|
|
88
|
-
ranked_attr = host_attrs[hk].keys.sort do |a,b|
|
81
|
+
ranked_attr = host_attrs[hk].keys.sort do |a,b|
|
89
82
|
host_attrs[hk][b] <=> host_attrs[hk][a]
|
90
83
|
end
|
91
84
|
result[hk] = ranked_attr.first
|
@@ -101,7 +94,7 @@ class Nizer
|
|
101
94
|
# matches within an os.product group. Multiple weak matches can
|
102
95
|
# outweigh a single strong match by design.
|
103
96
|
#
|
104
|
-
ranked_os = os_products.keys.sort do |a,b|
|
97
|
+
ranked_os = os_products.keys.sort do |a,b|
|
105
98
|
os_products[b].map{ |r| r['os.certainty'] }.inject(:+) <=>
|
106
99
|
os_products[a].map{ |r| r['os.certainty'] }.inject(:+)
|
107
100
|
end
|
@@ -151,11 +144,11 @@ class Nizer
|
|
151
144
|
end
|
152
145
|
|
153
146
|
#
|
154
|
-
# Select the best service name by combined certainty of all matches
|
155
|
-
# within an service.product group. Multiple weak matches can
|
147
|
+
# Select the best service name by combined certainty of all matches
|
148
|
+
# within an service.product group. Multiple weak matches can
|
156
149
|
# outweigh a single strong match by design.
|
157
150
|
#
|
158
|
-
ranked_service = service_products.keys.sort do |a,b|
|
151
|
+
ranked_service = service_products.keys.sort do |a,b|
|
159
152
|
service_products[b].map{ |r| r['service.certainty'] }.inject(:+) <=>
|
160
153
|
service_products[a].map{ |r| r['service.certainty'] }.inject(:+)
|
161
154
|
end
|
@@ -177,7 +170,7 @@ class Nizer
|
|
177
170
|
|
178
171
|
result
|
179
172
|
end
|
180
|
-
|
173
|
+
|
181
174
|
end
|
182
175
|
end
|
183
176
|
|
@@ -259,5 +252,5 @@ Current key names:
|
|
259
252
|
timeout
|
260
253
|
tomcat.info
|
261
254
|
zmailer.ident
|
262
|
-
|
263
|
-
=end
|
255
|
+
|
256
|
+
=end
|
data/lib/recog/verifier.rb
CHANGED
@@ -12,33 +12,18 @@ class Verifier
|
|
12
12
|
fingerprints.each do |fp|
|
13
13
|
reporter.print_name fp
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
failure = "'#{fp.name}' failed to match #{test.inspect} with #{fp.regex.to_s}'"
|
24
|
-
reporter.failure("FAIL: #{failure}")
|
25
|
-
else
|
26
|
-
info = { }
|
27
|
-
fp.params.each_pair do |k,v|
|
28
|
-
if v[0] == 0
|
29
|
-
info[k] = v[1]
|
30
|
-
else
|
31
|
-
info[k] = m[ v[0] ]
|
32
|
-
if m[ v[0] ].to_s.empty?
|
33
|
-
warning = "'#{fp.name}' failed to match #{test.inspect} key '#{k}'' with #{fp.regex.to_s}'"
|
34
|
-
reporter.warning "WARN: #{warning}"
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
reporter.success(test)
|
15
|
+
fp.verify_tests do |status, message|
|
16
|
+
case status
|
17
|
+
when :warn
|
18
|
+
reporter.warning "WARN: #{message}"
|
19
|
+
when :fail
|
20
|
+
reporter.failure "FAIL: #{message}"
|
21
|
+
when :success
|
22
|
+
reporter.success(message)
|
40
23
|
end
|
24
|
+
|
41
25
|
end
|
26
|
+
|
42
27
|
end
|
43
28
|
end
|
44
29
|
end
|
data/lib/recog/version.rb
CHANGED
data/recog.gemspec
CHANGED
@@ -14,9 +14,9 @@ Gem::Specification.new do |s|
|
|
14
14
|
s.homepage = "https://www.github.com/rapid7/recog"
|
15
15
|
s.summary = %q{Network service fingerprint database, classes, and utilities}
|
16
16
|
s.description = %q{
|
17
|
-
Recog is a framework for identifying products, services, operating systems, and hardware by matching
|
18
|
-
fingerprints against data returned from various network probes. Recog makes it simply to extract useful
|
19
|
-
information from web server banners, snmp system description fields, and a whole lot more.
|
17
|
+
Recog is a framework for identifying products, services, operating systems, and hardware by matching
|
18
|
+
fingerprints against data returned from various network probes. Recog makes it simply to extract useful
|
19
|
+
information from web server banners, snmp system description fields, and a whole lot more.
|
20
20
|
}.gsub(/\s+/, ' ').strip
|
21
21
|
|
22
22
|
s.files = `git ls-files`.split("\n")
|
@@ -27,8 +27,17 @@ Gem::Specification.new do |s|
|
|
27
27
|
# ---- Dependencies ----
|
28
28
|
|
29
29
|
s.add_development_dependency 'rspec'
|
30
|
+
s.add_development_dependency 'yard'
|
31
|
+
if RUBY_PLATFORM =~ /java/
|
32
|
+
# markdown formatting for yard
|
33
|
+
s.add_development_dependency 'kramdown'
|
34
|
+
else
|
35
|
+
# markdown formatting for yard
|
36
|
+
s.add_development_dependency 'redcarpet'
|
37
|
+
end
|
30
38
|
s.add_development_dependency 'cucumber'
|
31
39
|
s.add_development_dependency 'aruba'
|
40
|
+
s.add_development_dependency 'simplecov'
|
32
41
|
|
33
42
|
s.add_runtime_dependency 'nokogiri'
|
34
43
|
end
|
@@ -21,4 +21,16 @@
|
|
21
21
|
<example>HP LaserJet 2200</example>
|
22
22
|
<param pos="0" name="service.vendor" value="HP"/>
|
23
23
|
</fingerprint>
|
24
|
+
|
25
|
+
<fingerprint pattern="^Windows XP (\d+) (Service Pack \d+)$">
|
26
|
+
<description>Windows XP</description>
|
27
|
+
<example os.build="2600" os.version="Service Pack 1"
|
28
|
+
>Windows XP 2600 Service Pack 1</example>
|
29
|
+
<param pos="0" name="os.certainty" value="1.0"/>
|
30
|
+
<param pos="0" name="os.vendor" value="Microsoft"/>
|
31
|
+
<param pos="0" name="os.product" value="Windows XP"/>
|
32
|
+
<param pos="1" name="os.build"/>
|
33
|
+
<param pos="2" name="os.version"/>
|
34
|
+
</fingerprint>
|
35
|
+
|
24
36
|
</fingerprints>
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
require 'recog/db'
|
2
2
|
|
3
3
|
describe Recog::DB do
|
4
4
|
Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
|
@@ -16,6 +16,10 @@ describe Recog::DB do
|
|
16
16
|
|
17
17
|
context "#{fp.regex}" do
|
18
18
|
|
19
|
+
if fp.name.nil? || fp.name.empty?
|
20
|
+
skip "has a name"
|
21
|
+
end
|
22
|
+
|
19
23
|
# Not yet enforced
|
20
24
|
# it "has a name" do
|
21
25
|
# expect(fp.name).not_to be_nil
|
@@ -28,13 +32,13 @@ describe Recog::DB do
|
|
28
32
|
end
|
29
33
|
|
30
34
|
# Not yet enforced
|
31
|
-
# it "has
|
35
|
+
# it "has test cases" do
|
32
36
|
# expect(fp.tests.length).not_to equal(0)
|
33
37
|
# end
|
34
38
|
|
35
39
|
fp.tests.each do |example|
|
36
|
-
it "passes self-test #{example.gsub(/\s+/, ' ')[0,32]}..." do
|
37
|
-
expect(fp.
|
40
|
+
it "passes self-test #{example.content.gsub(/\s+/, ' ')[0,32]}..." do
|
41
|
+
expect(fp.match(example.content)).to_not be_nil
|
38
42
|
end
|
39
43
|
end
|
40
44
|
|
@@ -1,12 +1,16 @@
|
|
1
|
-
|
1
|
+
require 'recog/db'
|
2
2
|
|
3
3
|
describe Recog::DB do
|
4
4
|
let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
|
5
5
|
subject { Recog::DB.new(xml_file) }
|
6
6
|
|
7
7
|
describe "#fingerprints" do
|
8
|
+
subject(:fingerprints) { described_class.new(xml_file).fingerprints }
|
9
|
+
|
10
|
+
it { is_expected.to be_a(Enumerable) }
|
11
|
+
|
8
12
|
context "with only a pattern" do
|
9
|
-
|
13
|
+
subject(:entry) { described_class.new(xml_file).fingerprints[0] }
|
10
14
|
|
11
15
|
it "has a blank name with no description" do
|
12
16
|
expect(entry.name).to be_empty
|
@@ -26,7 +30,7 @@ describe Recog::DB do
|
|
26
30
|
end
|
27
31
|
|
28
32
|
context "with params" do
|
29
|
-
|
33
|
+
subject(:entry) { described_class.new(xml_file).fingerprints[1] }
|
30
34
|
|
31
35
|
it "has a name" do
|
32
36
|
expect(entry.name).to eq('PalmOS')
|
@@ -46,14 +50,22 @@ describe Recog::DB do
|
|
46
50
|
end
|
47
51
|
|
48
52
|
context "with pattern flags" do
|
49
|
-
|
53
|
+
subject(:entry) { described_class.new(xml_file).fingerprints[2] }
|
50
54
|
|
51
55
|
it "has a name and only uses the first value" do
|
52
56
|
expect(entry.name).to eq('HP Designjet printer')
|
53
57
|
end
|
54
58
|
|
55
|
-
it
|
59
|
+
it 'creates a Regexp with expected flags' do
|
60
|
+
if RUBY_PLATFORM =~ /java/i
|
61
|
+
pending "Bug in jruby"
|
62
|
+
end
|
63
|
+
expect(entry.regex).to be_a(Regexp)
|
56
64
|
expect(entry.regex.options).to eq(Regexp::NOENCODING | Regexp::IGNORECASE)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "has a pattern" do
|
68
|
+
expect(entry.regex).to be_a(Regexp)
|
57
69
|
expect(entry.regex.source).to eq("(designjet \\S+)")
|
58
70
|
end
|
59
71
|
|
@@ -67,7 +79,7 @@ describe Recog::DB do
|
|
67
79
|
end
|
68
80
|
|
69
81
|
context "with test" do
|
70
|
-
|
82
|
+
subject(:entry) { described_class.new(xml_file).fingerprints[3] }
|
71
83
|
|
72
84
|
it "has a name" do
|
73
85
|
expect(entry.name).to eq('HP JetDirect Printer')
|
@@ -82,7 +94,7 @@ describe Recog::DB do
|
|
82
94
|
end
|
83
95
|
|
84
96
|
it "has no tests" do
|
85
|
-
expect(entry.tests).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
|
97
|
+
expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
|
86
98
|
end
|
87
99
|
end
|
88
100
|
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
|
2
|
+
require 'recog/fingerprint/regexp_factory'
|
3
|
+
|
4
|
+
describe Recog::Fingerprint::RegexpFactory do
|
5
|
+
|
6
|
+
describe 'FLAG_MAP' do
|
7
|
+
subject { described_class::FLAG_MAP }
|
8
|
+
|
9
|
+
it "should have three flags" do
|
10
|
+
expect(subject.size).to be 3
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe '.build' do
|
15
|
+
subject { described_class.build(pattern, options) }
|
16
|
+
|
17
|
+
let(:pattern) { 'Apache/(\d+)' }
|
18
|
+
let(:options) { [ 'REG_ICASE' ] }
|
19
|
+
|
20
|
+
it { is_expected.to be_a(Regexp) }
|
21
|
+
it { is_expected.to match('Apache/2') }
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '.build_options' do
|
26
|
+
subject { described_class.build_options(flags) }
|
27
|
+
|
28
|
+
let(:flags) { [ ] }
|
29
|
+
it { is_expected.to be_a(Fixnum) }
|
30
|
+
|
31
|
+
specify "sets NOENCODING" do
|
32
|
+
expect(subject & Regexp::NOENCODING).to_not be_zero
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'with REG_ICASE' do
|
36
|
+
let(:flags) { [ 'REG_ICASE' ] }
|
37
|
+
specify "sets NOENCODING & IGNORECASE" do
|
38
|
+
expect(subject & Regexp::NOENCODING).to_not be_zero
|
39
|
+
expect(subject & Regexp::IGNORECASE).to_not be_zero
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'with REG_DOT_NEWLINE' do
|
44
|
+
let(:flags) { [ 'REG_DOT_NEWLINE' ] }
|
45
|
+
specify "sets NOENCODING & MULTILINE" do
|
46
|
+
expect(subject & Regexp::NOENCODING).to_not be_zero
|
47
|
+
expect(subject & Regexp::MULTILINE).to_not be_zero
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'with REG_LINE_ANY_CRLF' do
|
52
|
+
let(:flags) { [ 'REG_LINE_ANY_CRLF' ] }
|
53
|
+
specify "sets NOENCODING & MULTILINE" do
|
54
|
+
expect(subject & Regexp::NOENCODING).to_not be_zero
|
55
|
+
expect(subject & Regexp::MULTILINE).to_not be_zero
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|