recog 0.02 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -1
  4. data/.travis.yml +5 -0
  5. data/.yardopts +1 -0
  6. data/Gemfile +3 -1
  7. data/README.md +12 -12
  8. data/Rakefile +22 -0
  9. data/bin/recog_verify.rb +1 -1
  10. data/features/match.feature +2 -2
  11. data/features/verify.feature +10 -7
  12. data/features/xml/no_tests.xml +0 -50
  13. data/features/xml/successful_tests.xml +7 -22
  14. data/features/xml/tests_with_failures.xml +10 -0
  15. data/features/xml/tests_with_warnings.xml +7 -0
  16. data/lib/recog/db.rb +26 -10
  17. data/lib/recog/db_manager.rb +1 -1
  18. data/lib/recog/fingerprint.rb +118 -34
  19. data/lib/recog/fingerprint/regexp_factory.rb +39 -0
  20. data/lib/recog/fingerprint/test.rb +13 -0
  21. data/lib/recog/matcher.rb +3 -3
  22. data/lib/recog/nizer.rb +16 -23
  23. data/lib/recog/verifier.rb +10 -25
  24. data/lib/recog/verifier_factory.rb +1 -1
  25. data/lib/recog/verify_reporter.rb +1 -1
  26. data/lib/recog/version.rb +1 -1
  27. data/recog.gemspec +12 -3
  28. data/spec/data/test_fingerprints.xml +12 -0
  29. data/spec/lib/fingerprint_self_test_spec.rb +8 -4
  30. data/spec/lib/{db_spec.rb → recog/db_spec.rb} +19 -7
  31. data/spec/lib/recog/fingerprint/regexp_factory.rb +61 -0
  32. data/spec/lib/recog/fingerprint_spec.rb +5 -0
  33. data/spec/lib/{formatter_spec.rb → recog/formatter_spec.rb} +1 -1
  34. data/spec/lib/{match_reporter_spec.rb → recog/match_reporter_spec.rb} +10 -9
  35. data/spec/lib/{nizer_spec.rb → recog/nizer_spec.rb} +5 -5
  36. data/spec/lib/{verify_reporter_spec.rb → recog/verify_reporter_spec.rb} +8 -7
  37. data/spec/spec_helper.rb +82 -0
  38. data/xml/apache_os.xml +48 -2
  39. data/xml/http_servers.xml +38 -6
  40. data/xml/ntp_banners.xml +4 -3
  41. data/xml/smb_native_os.xml +32 -32
  42. data/xml/smtp_expn.xml +1 -0
  43. data/xml/smtp_help.xml +2 -1
  44. data/xml/snmp_sysdescr.xml +164 -24
  45. data/xml/ssh_banners.xml +7 -3
  46. metadata +56 -8
  47. data/Gemfile.lock +0 -42
@@ -0,0 +1,39 @@
1
+
2
+ module Recog
3
+ class Fingerprint
4
+
5
+ #
6
+ # @example
7
+ # r = RegexpFactory.build("^Apache[ -]Coyote/(\d\.\d)$", "REG_ICASE")
8
+ # r.match("Apache-Coyote/1.1")
9
+ #
10
+ module RegexpFactory
11
+
12
+ # Map strings as used in Recog XML to Fixnum values used by Regexp
13
+ FLAG_MAP = {
14
+ 'REG_DOT_NEWLINE' => Regexp::MULTILINE,
15
+ 'REG_LINE_ANY_CRLF' => Regexp::MULTILINE,
16
+ 'REG_ICASE' => Regexp::IGNORECASE,
17
+ }
18
+
19
+ # @return [Regexp]
20
+ def self.build(pattern, flags)
21
+ options = build_options(flags)
22
+ Regexp.new(pattern, options)
23
+ end
24
+
25
+ # Convert string flag names as used in Recog XML into a Fixnum suitable for
26
+ # passing as the `options` parameter to `Regexp.new`
27
+ #
28
+ # @see FLAG_MAP
29
+ # @param flags [Array<String>]
30
+ # @return [Fixnum] Flags for creating a regular expression object
31
+ def self.build_options(flags)
32
+ flags.reduce(Regexp::NOENCODING) do |sum, flag|
33
+ sum |= (FLAG_MAP[flag] || 0)
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
39
+
@@ -0,0 +1,13 @@
1
+
2
+ class Recog::Fingerprint::Test
3
+ attr_accessor :content
4
+ attr_accessor :attributes
5
+ def initialize(content, attributes=[])
6
+ @content = content
7
+ @attributes = attributes
8
+ end
9
+
10
+ def to_s
11
+ content
12
+ end
13
+ end
data/lib/recog/matcher.rb CHANGED
@@ -9,7 +9,7 @@ class Matcher
9
9
 
10
10
  def match_banners(banners_file)
11
11
  reporter.report do
12
-
12
+
13
13
  fd = $stdin
14
14
  file_source = false
15
15
 
@@ -22,7 +22,7 @@ class Matcher
22
22
  reporter.increment_line_count
23
23
 
24
24
  line = line.to_s.unpack("C*").pack("C*").strip.gsub(/\\[rn]/, '')
25
- found = nil
25
+ found = nil
26
26
  fingerprints.each do |fp|
27
27
  m = line.match(fp.regex)
28
28
  if m
@@ -53,7 +53,7 @@ class Matcher
53
53
  end
54
54
 
55
55
  fd.close if file_source
56
-
56
+
57
57
  end
58
58
  end
59
59
  end
data/lib/recog/nizer.rb CHANGED
@@ -22,27 +22,20 @@ class Nizer
22
22
  @@db_manager = nil
23
23
 
24
24
  #
25
- # Locate a database that corresponds with the match_key and attempt to
26
- # find a matching fingerprinting, stopping a the first hit. Returns
27
- # nil when no matching database or fingerprint is found.
25
+ # Locate a database that corresponds with the `match_key` and attempt to
26
+ # find a matching {Fingerprint fingerprint}, stopping at the first hit. Returns `nil`
27
+ # when no matching database or fingerprint is found.
28
28
  #
29
+ # @param match_key [String] Fingerprint DB name, e.g. 'smb.native_os'
30
+ # @return (see Fingerprint#match)
29
31
  def self.match(match_key, match_string)
30
32
  match_string = match_string.to_s.unpack("C*").pack("C*")
31
33
  @@db_manager ||= Recog::DBManager.new
32
34
  @@db_manager.databases.each do |db|
33
35
  next unless db.match_key == match_key
34
36
  db.fingerprints.each do |fprint|
35
- m = fprint.regex.match(match_string)
36
- next unless m
37
- result = { 'matched' => fprint.name }
38
- fprint.params.each_pair do |k,v|
39
- if v[0] == 0
40
- result[k] = v[1]
41
- else
42
- result[k] = m[ v[0] ]
43
- end
44
- end
45
- return result
37
+ m = fprint.match(match_string)
38
+ return m if m
46
39
  end
47
40
  end
48
41
  nil
@@ -69,7 +62,7 @@ class Nizer
69
62
  (HOST_ATTRIBUTES & m.keys).each do |ha|
70
63
  host_attrs[ha] ||= {}
71
64
  host_attrs[ha][m[ha]] ||= 0
72
- host_attrs[ha][m[ha]] += 1
65
+ host_attrs[ha][m[ha]] += 1
73
66
  end
74
67
 
75
68
  next unless m.has_key?('os.product')
@@ -85,7 +78,7 @@ class Nizer
85
78
  # Select the best host attribute value by highest frequency
86
79
  #
87
80
  host_attrs.keys.each do |hk|
88
- ranked_attr = host_attrs[hk].keys.sort do |a,b|
81
+ ranked_attr = host_attrs[hk].keys.sort do |a,b|
89
82
  host_attrs[hk][b] <=> host_attrs[hk][a]
90
83
  end
91
84
  result[hk] = ranked_attr.first
@@ -101,7 +94,7 @@ class Nizer
101
94
  # matches within an os.product group. Multiple weak matches can
102
95
  # outweigh a single strong match by design.
103
96
  #
104
- ranked_os = os_products.keys.sort do |a,b|
97
+ ranked_os = os_products.keys.sort do |a,b|
105
98
  os_products[b].map{ |r| r['os.certainty'] }.inject(:+) <=>
106
99
  os_products[a].map{ |r| r['os.certainty'] }.inject(:+)
107
100
  end
@@ -151,11 +144,11 @@ class Nizer
151
144
  end
152
145
 
153
146
  #
154
- # Select the best service name by combined certainty of all matches
155
- # within an service.product group. Multiple weak matches can
147
+ # Select the best service name by combined certainty of all matches
148
+ # within an service.product group. Multiple weak matches can
156
149
  # outweigh a single strong match by design.
157
150
  #
158
- ranked_service = service_products.keys.sort do |a,b|
151
+ ranked_service = service_products.keys.sort do |a,b|
159
152
  service_products[b].map{ |r| r['service.certainty'] }.inject(:+) <=>
160
153
  service_products[a].map{ |r| r['service.certainty'] }.inject(:+)
161
154
  end
@@ -177,7 +170,7 @@ class Nizer
177
170
 
178
171
  result
179
172
  end
180
-
173
+
181
174
  end
182
175
  end
183
176
 
@@ -259,5 +252,5 @@ Current key names:
259
252
  timeout
260
253
  tomcat.info
261
254
  zmailer.ident
262
-
263
- =end
255
+
256
+ =end
@@ -12,33 +12,18 @@ class Verifier
12
12
  fingerprints.each do |fp|
13
13
  reporter.print_name fp
14
14
 
15
- if fp.tests.length == 0
16
- warning = "'#{fp.name}' has no test cases"
17
- reporter.warning "WARN: #{warning}"
18
- end
19
-
20
- fp.tests.each do |test|
21
- m = test.match(fp.regex)
22
- unless m
23
- failure = "'#{fp.name}' failed to match #{test.inspect} with #{fp.regex.to_s}'"
24
- reporter.failure("FAIL: #{failure}")
25
- else
26
- info = { }
27
- fp.params.each_pair do |k,v|
28
- if v[0] == 0
29
- info[k] = v[1]
30
- else
31
- info[k] = m[ v[0] ]
32
- if m[ v[0] ].to_s.empty?
33
- warning = "'#{fp.name}' failed to match #{test.inspect} key '#{k}'' with #{fp.regex.to_s}'"
34
- reporter.warning "WARN: #{warning}"
35
- end
36
- end
37
- end
38
-
39
- reporter.success(test)
15
+ fp.verify_tests do |status, message|
16
+ case status
17
+ when :warn
18
+ reporter.warning "WARN: #{message}"
19
+ when :fail
20
+ reporter.failure "FAIL: #{message}"
21
+ when :success
22
+ reporter.success(message)
40
23
  end
24
+
41
25
  end
26
+
42
27
  end
43
28
  end
44
29
  end
@@ -10,4 +10,4 @@ module VerifierFactory
10
10
  Verifier.new(options.fingerprints, reporter)
11
11
  end
12
12
  end
13
- end
13
+ end
@@ -82,4 +82,4 @@ class VerifyReporter
82
82
  end
83
83
  end
84
84
  end
85
- end
85
+ end
data/lib/recog/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Recog
2
- VERSION = "0.02"
2
+ VERSION = "1.0.0"
3
3
  end
data/recog.gemspec CHANGED
@@ -14,9 +14,9 @@ Gem::Specification.new do |s|
14
14
  s.homepage = "https://www.github.com/rapid7/recog"
15
15
  s.summary = %q{Network service fingerprint database, classes, and utilities}
16
16
  s.description = %q{
17
- Recog is a framework for identifying products, services, operating systems, and hardware by matching
18
- fingerprints against data returned from various network probes. Recog makes it simply to extract useful
19
- information from web server banners, snmp system description fields, and a whole lot more.
17
+ Recog is a framework for identifying products, services, operating systems, and hardware by matching
18
+ fingerprints against data returned from various network probes. Recog makes it simply to extract useful
19
+ information from web server banners, snmp system description fields, and a whole lot more.
20
20
  }.gsub(/\s+/, ' ').strip
21
21
 
22
22
  s.files = `git ls-files`.split("\n")
@@ -27,8 +27,17 @@ Gem::Specification.new do |s|
27
27
  # ---- Dependencies ----
28
28
 
29
29
  s.add_development_dependency 'rspec'
30
+ s.add_development_dependency 'yard'
31
+ if RUBY_PLATFORM =~ /java/
32
+ # markdown formatting for yard
33
+ s.add_development_dependency 'kramdown'
34
+ else
35
+ # markdown formatting for yard
36
+ s.add_development_dependency 'redcarpet'
37
+ end
30
38
  s.add_development_dependency 'cucumber'
31
39
  s.add_development_dependency 'aruba'
40
+ s.add_development_dependency 'simplecov'
32
41
 
33
42
  s.add_runtime_dependency 'nokogiri'
34
43
  end
@@ -21,4 +21,16 @@
21
21
  <example>HP LaserJet 2200</example>
22
22
  <param pos="0" name="service.vendor" value="HP"/>
23
23
  </fingerprint>
24
+
25
+ <fingerprint pattern="^Windows XP (\d+) (Service Pack \d+)$">
26
+ <description>Windows XP</description>
27
+ <example os.build="2600" os.version="Service Pack 1"
28
+ >Windows XP 2600 Service Pack 1</example>
29
+ <param pos="0" name="os.certainty" value="1.0"/>
30
+ <param pos="0" name="os.vendor" value="Microsoft"/>
31
+ <param pos="0" name="os.product" value="Windows XP"/>
32
+ <param pos="1" name="os.build"/>
33
+ <param pos="2" name="os.version"/>
34
+ </fingerprint>
35
+
24
36
  </fingerprints>
@@ -1,4 +1,4 @@
1
- require_relative '../../lib/recog/db'
1
+ require 'recog/db'
2
2
 
3
3
  describe Recog::DB do
4
4
  Dir[File.expand_path File.join('xml', '*.xml')].each do |xml_file_name|
@@ -16,6 +16,10 @@ describe Recog::DB do
16
16
 
17
17
  context "#{fp.regex}" do
18
18
 
19
+ if fp.name.nil? || fp.name.empty?
20
+ skip "has a name"
21
+ end
22
+
19
23
  # Not yet enforced
20
24
  # it "has a name" do
21
25
  # expect(fp.name).not_to be_nil
@@ -28,13 +32,13 @@ describe Recog::DB do
28
32
  end
29
33
 
30
34
  # Not yet enforced
31
- # it "has a test cases" do
35
+ # it "has test cases" do
32
36
  # expect(fp.tests.length).not_to equal(0)
33
37
  # end
34
38
 
35
39
  fp.tests.each do |example|
36
- it "passes self-test #{example.gsub(/\s+/, ' ')[0,32]}..." do
37
- expect(fp.regex.match(example)).to_not eq(nil)
40
+ it "passes self-test #{example.content.gsub(/\s+/, ' ')[0,32]}..." do
41
+ expect(fp.match(example.content)).to_not be_nil
38
42
  end
39
43
  end
40
44
 
@@ -1,12 +1,16 @@
1
- require_relative '../../lib/recog/db'
1
+ require 'recog/db'
2
2
 
3
3
  describe Recog::DB do
4
4
  let(:xml_file) { File.expand_path File.join('spec', 'data', 'test_fingerprints.xml') }
5
5
  subject { Recog::DB.new(xml_file) }
6
6
 
7
7
  describe "#fingerprints" do
8
+ subject(:fingerprints) { described_class.new(xml_file).fingerprints }
9
+
10
+ it { is_expected.to be_a(Enumerable) }
11
+
8
12
  context "with only a pattern" do
9
- let(:entry) { subject.fingerprints[0] }
13
+ subject(:entry) { described_class.new(xml_file).fingerprints[0] }
10
14
 
11
15
  it "has a blank name with no description" do
12
16
  expect(entry.name).to be_empty
@@ -26,7 +30,7 @@ describe Recog::DB do
26
30
  end
27
31
 
28
32
  context "with params" do
29
- let(:entry) { subject.fingerprints[1] }
33
+ subject(:entry) { described_class.new(xml_file).fingerprints[1] }
30
34
 
31
35
  it "has a name" do
32
36
  expect(entry.name).to eq('PalmOS')
@@ -46,14 +50,22 @@ describe Recog::DB do
46
50
  end
47
51
 
48
52
  context "with pattern flags" do
49
- let(:entry) { subject.fingerprints[2] }
53
+ subject(:entry) { described_class.new(xml_file).fingerprints[2] }
50
54
 
51
55
  it "has a name and only uses the first value" do
52
56
  expect(entry.name).to eq('HP Designjet printer')
53
57
  end
54
58
 
55
- it "has a pattern" do
59
+ it 'creates a Regexp with expected flags' do
60
+ if RUBY_PLATFORM =~ /java/i
61
+ pending "Bug in jruby"
62
+ end
63
+ expect(entry.regex).to be_a(Regexp)
56
64
  expect(entry.regex.options).to eq(Regexp::NOENCODING | Regexp::IGNORECASE)
65
+ end
66
+
67
+ it "has a pattern" do
68
+ expect(entry.regex).to be_a(Regexp)
57
69
  expect(entry.regex.source).to eq("(designjet \\S+)")
58
70
  end
59
71
 
@@ -67,7 +79,7 @@ describe Recog::DB do
67
79
  end
68
80
 
69
81
  context "with test" do
70
- let(:entry) { subject.fingerprints[3] }
82
+ subject(:entry) { described_class.new(xml_file).fingerprints[3] }
71
83
 
72
84
  it "has a name" do
73
85
  expect(entry.name).to eq('HP JetDirect Printer')
@@ -82,7 +94,7 @@ describe Recog::DB do
82
94
  end
83
95
 
84
96
  it "has no tests" do
85
- expect(entry.tests).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
97
+ expect(entry.tests.map(&:content)).to match_array(["HP LaserJet 4100 Series", "HP LaserJet 2200"])
86
98
  end
87
99
  end
88
100
  end
@@ -0,0 +1,61 @@
1
+
2
+ require 'recog/fingerprint/regexp_factory'
3
+
4
+ describe Recog::Fingerprint::RegexpFactory do
5
+
6
+ describe 'FLAG_MAP' do
7
+ subject { described_class::FLAG_MAP }
8
+
9
+ it "should have three flags" do
10
+ expect(subject.size).to be 3
11
+ end
12
+ end
13
+
14
+ describe '.build' do
15
+ subject { described_class.build(pattern, options) }
16
+
17
+ let(:pattern) { 'Apache/(\d+)' }
18
+ let(:options) { [ 'REG_ICASE' ] }
19
+
20
+ it { is_expected.to be_a(Regexp) }
21
+ it { is_expected.to match('Apache/2') }
22
+
23
+ end
24
+
25
+ describe '.build_options' do
26
+ subject { described_class.build_options(flags) }
27
+
28
+ let(:flags) { [ ] }
29
+ it { is_expected.to be_a(Fixnum) }
30
+
31
+ specify "sets NOENCODING" do
32
+ expect(subject & Regexp::NOENCODING).to_not be_zero
33
+ end
34
+
35
+ context 'with REG_ICASE' do
36
+ let(:flags) { [ 'REG_ICASE' ] }
37
+ specify "sets NOENCODING & IGNORECASE" do
38
+ expect(subject & Regexp::NOENCODING).to_not be_zero
39
+ expect(subject & Regexp::IGNORECASE).to_not be_zero
40
+ end
41
+ end
42
+
43
+ context 'with REG_DOT_NEWLINE' do
44
+ let(:flags) { [ 'REG_DOT_NEWLINE' ] }
45
+ specify "sets NOENCODING & MULTILINE" do
46
+ expect(subject & Regexp::NOENCODING).to_not be_zero
47
+ expect(subject & Regexp::MULTILINE).to_not be_zero
48
+ end
49
+ end
50
+
51
+ context 'with REG_LINE_ANY_CRLF' do
52
+ let(:flags) { [ 'REG_LINE_ANY_CRLF' ] }
53
+ specify "sets NOENCODING & MULTILINE" do
54
+ expect(subject & Regexp::NOENCODING).to_not be_zero
55
+ expect(subject & Regexp::MULTILINE).to_not be_zero
56
+ end
57
+ end
58
+
59
+ end
60
+ end
61
+