recog-intrigue 2.3.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (130) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +17 -0
  4. data/.github/ISSUE_TEMPLATE/fingerprint_request.md +27 -0
  5. data/.github/PULL_REQUEST_TEMPLATE +24 -0
  6. data/.gitignore +14 -0
  7. data/.rbenv-gemset +1 -0
  8. data/.rspec +3 -0
  9. data/.ruby-gemset +1 -0
  10. data/.ruby-version +1 -0
  11. data/.travis.yml +25 -0
  12. data/.yardopts +1 -0
  13. data/CONTRIBUTING.md +171 -0
  14. data/COPYING +23 -0
  15. data/Gemfile +10 -0
  16. data/LICENSE +7 -0
  17. data/README.md +85 -0
  18. data/Rakefile +22 -0
  19. data/bin/recog_export +81 -0
  20. data/bin/recog_match +55 -0
  21. data/bin/recog_standardize +118 -0
  22. data/bin/recog_verify +64 -0
  23. data/cpe-remap.yaml +134 -0
  24. data/features/data/failing_banners_fingerprints.xml +20 -0
  25. data/features/data/matching_banners_fingerprints.xml +23 -0
  26. data/features/data/multiple_banners_fingerprints.xml +32 -0
  27. data/features/data/no_tests.xml +3 -0
  28. data/features/data/sample_banner.txt +2 -0
  29. data/features/data/successful_tests.xml +18 -0
  30. data/features/data/tests_with_failures.xml +20 -0
  31. data/features/data/tests_with_warnings.xml +17 -0
  32. data/features/match.feature +36 -0
  33. data/features/support/aruba.rb +3 -0
  34. data/features/support/env.rb +6 -0
  35. data/features/verify.feature +48 -0
  36. data/identifiers/README.md +47 -0
  37. data/identifiers/os_architecture.txt +20 -0
  38. data/identifiers/os_device.txt +52 -0
  39. data/identifiers/os_family.txt +160 -0
  40. data/identifiers/os_product.txt +199 -0
  41. data/identifiers/service_family.txt +185 -0
  42. data/identifiers/service_product.txt +255 -0
  43. data/identifiers/software_class.txt +26 -0
  44. data/identifiers/software_family.txt +91 -0
  45. data/identifiers/software_product.txt +333 -0
  46. data/identifiers/vendor.txt +405 -0
  47. data/lib/recog.rb +4 -0
  48. data/lib/recog/db.rb +78 -0
  49. data/lib/recog/db_manager.rb +31 -0
  50. data/lib/recog/fingerprint.rb +280 -0
  51. data/lib/recog/fingerprint/regexp_factory.rb +56 -0
  52. data/lib/recog/fingerprint/test.rb +18 -0
  53. data/lib/recog/formatter.rb +51 -0
  54. data/lib/recog/match_reporter.rb +77 -0
  55. data/lib/recog/matcher.rb +94 -0
  56. data/lib/recog/matcher_factory.rb +14 -0
  57. data/lib/recog/nizer.rb +347 -0
  58. data/lib/recog/verifier.rb +39 -0
  59. data/lib/recog/verifier_factory.rb +13 -0
  60. data/lib/recog/verify_reporter.rb +86 -0
  61. data/lib/recog/version.rb +3 -0
  62. data/misc/convert_mysql_err +61 -0
  63. data/misc/order.xsl +17 -0
  64. data/recog-intrigue.gemspec +45 -0
  65. data/requirements.txt +2 -0
  66. data/spec/data/best_os_match_1.yml +17 -0
  67. data/spec/data/best_os_match_2.yml +17 -0
  68. data/spec/data/best_service_match_1.yml +17 -0
  69. data/spec/data/smb_native_os.txt +25 -0
  70. data/spec/data/test_fingerprints.xml +36 -0
  71. data/spec/data/verification_fingerprints.xml +86 -0
  72. data/spec/data/whitespaced_fingerprint.xml +5 -0
  73. data/spec/lib/fingerprint_self_test_spec.rb +174 -0
  74. data/spec/lib/recog/db_spec.rb +98 -0
  75. data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +73 -0
  76. data/spec/lib/recog/fingerprint_spec.rb +112 -0
  77. data/spec/lib/recog/formatter_spec.rb +69 -0
  78. data/spec/lib/recog/match_reporter_spec.rb +91 -0
  79. data/spec/lib/recog/nizer_spec.rb +330 -0
  80. data/spec/lib/recog/verify_reporter_spec.rb +113 -0
  81. data/spec/spec_helper.rb +82 -0
  82. data/update_cpes.py +186 -0
  83. data/xml/apache_modules.xml +1911 -0
  84. data/xml/apache_os.xml +273 -0
  85. data/xml/architecture.xml +36 -0
  86. data/xml/dns_versionbind.xml +761 -0
  87. data/xml/fingerprints.xsd +128 -0
  88. data/xml/ftp_banners.xml +1553 -0
  89. data/xml/h323_callresp.xml +603 -0
  90. data/xml/hp_pjl_id.xml +358 -0
  91. data/xml/html_title.xml +1630 -0
  92. data/xml/http_cookies.xml +411 -0
  93. data/xml/http_servers.xml +3195 -0
  94. data/xml/http_wwwauth.xml +595 -0
  95. data/xml/imap_banners.xml +245 -0
  96. data/xml/ldap_searchresult.xml +711 -0
  97. data/xml/mdns_device-info_txt.xml +1796 -0
  98. data/xml/mdns_workstation_txt.xml +15 -0
  99. data/xml/mysql_banners.xml +1649 -0
  100. data/xml/mysql_error.xml +871 -0
  101. data/xml/nntp_banners.xml +82 -0
  102. data/xml/ntp_banners.xml +1223 -0
  103. data/xml/operating_system.xml +629 -0
  104. data/xml/pop_banners.xml +499 -0
  105. data/xml/rsh_resp.xml +76 -0
  106. data/xml/rtsp_servers.xml +76 -0
  107. data/xml/sip_banners.xml +359 -0
  108. data/xml/sip_user_agents.xml +221 -0
  109. data/xml/smb_native_lm.xml +62 -0
  110. data/xml/smb_native_os.xml +662 -0
  111. data/xml/smtp_banners.xml +1690 -0
  112. data/xml/smtp_debug.xml +39 -0
  113. data/xml/smtp_ehlo.xml +49 -0
  114. data/xml/smtp_expn.xml +82 -0
  115. data/xml/smtp_help.xml +157 -0
  116. data/xml/smtp_mailfrom.xml +20 -0
  117. data/xml/smtp_noop.xml +44 -0
  118. data/xml/smtp_quit.xml +29 -0
  119. data/xml/smtp_rcptto.xml +25 -0
  120. data/xml/smtp_rset.xml +26 -0
  121. data/xml/smtp_turn.xml +26 -0
  122. data/xml/smtp_vrfy.xml +89 -0
  123. data/xml/snmp_sysdescr.xml +6507 -0
  124. data/xml/snmp_sysobjid.xml +430 -0
  125. data/xml/ssh_banners.xml +1968 -0
  126. data/xml/telnet_banners.xml +1595 -0
  127. data/xml/x11_banners.xml +232 -0
  128. data/xml/x509_issuers.xml +134 -0
  129. data/xml/x509_subjects.xml +1268 -0
  130. metadata +304 -0
@@ -0,0 +1,69 @@
1
+ require 'recog/formatter'
2
+
3
+ describe Recog::Formatter do
4
+ let(:output) { StringIO.new }
5
+
6
+ context "with no color" do
7
+ subject { Recog::Formatter.new(double(color: false), output) }
8
+
9
+ describe "#message" do
10
+ it "outputs the text" do
11
+ subject.status_message 'some text'
12
+ expect(output.string).to eq("some text\n")
13
+ end
14
+ end
15
+
16
+ describe "#success_message" do
17
+ it "outputs the text" do
18
+ subject.success_message 'a success'
19
+ expect(output.string).to eq("a success\n")
20
+ end
21
+ end
22
+
23
+ describe "#warning_message" do
24
+ it "outputs the text" do
25
+ subject.warning_message 'a warning'
26
+ expect(output.string).to eq("a warning\n")
27
+ end
28
+ end
29
+
30
+ describe "#failure_message" do
31
+ it "outputs the text" do
32
+ subject.failure_message 'a failure'
33
+ expect(output.string).to eq("a failure\n")
34
+ end
35
+ end
36
+ end
37
+
38
+ context "with color" do
39
+ subject { Recog::Formatter.new(double(color: true), output) }
40
+
41
+ describe "#message" do
42
+ it "outputs the text in white" do
43
+ subject.status_message 'some text'
44
+ expect(output.string).to eq("\e[15msome text\e[0m\n")
45
+ end
46
+ end
47
+
48
+ describe "#success_message" do
49
+ it "outputs the text in green" do
50
+ subject.success_message 'a success'
51
+ expect(output.string).to eq("\e[32ma success\e[0m\n")
52
+ end
53
+ end
54
+
55
+ describe "#warning_message" do
56
+ it "outputs the text in yellow" do
57
+ subject.warning_message 'a warning'
58
+ expect(output.string).to eq("\e[33ma warning\e[0m\n")
59
+ end
60
+ end
61
+
62
+ describe "#failure_message" do
63
+ it "outputs the text in red" do
64
+ subject.failure_message 'a failure'
65
+ expect(output.string).to eq("\e[31ma failure\e[0m\n")
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,91 @@
1
+ require 'recog/match_reporter'
2
+
3
+ describe Recog::MatchReporter do
4
+ let(:options) { double(detail: false, quiet: false) }
5
+ let(:formatter) { double('formatter').as_null_object }
6
+ subject { Recog::MatchReporter.new(options, formatter) }
7
+
8
+ def run_report
9
+ subject.report do
10
+ subject.increment_line_count
11
+ subject.match 'a match'
12
+ subject.failure 'a failure'
13
+ end
14
+ end
15
+
16
+ describe "#report" do
17
+ it "prints matches" do
18
+ expect(formatter).to receive(:success_message).with('a match')
19
+ run_report
20
+ end
21
+
22
+ it "prints failures" do
23
+ expect(formatter).to receive(:failure_message).with('a failure')
24
+ run_report
25
+ end
26
+
27
+ context "with detail" do
28
+ subject { Recog::MatchReporter.new(double(detail: true, quiet: false), formatter) }
29
+
30
+ it "prints the lines processed" do
31
+ expect(formatter).to receive(:status_message).with("\nProcessed 1 lines")
32
+ run_report
33
+ end
34
+
35
+ it "prints summary" do
36
+ expect(formatter).to receive(:failure_message).with("SUMMARY: 1 matches and 1 failures")
37
+ run_report
38
+ end
39
+ end
40
+ end
41
+
42
+ describe "#print_summary" do
43
+ context "with all matches" do
44
+ before { subject.match 'match' }
45
+
46
+ it "prints a successful summary" do
47
+ msg = "SUMMARY: 1 matches and 0 failures"
48
+ expect(formatter).to receive(:success_message).with(msg)
49
+ subject.print_summary
50
+ end
51
+ end
52
+
53
+ context "with failures" do
54
+ before { subject.failure 'fail' }
55
+
56
+ it "prints a failure summary" do
57
+ msg = "SUMMARY: 0 matches and 1 failures"
58
+ expect(formatter).to receive(:failure_message).with(msg)
59
+ subject.print_summary
60
+ end
61
+ end
62
+ end
63
+
64
+ describe "#stop?" do
65
+ context "with a failure limit" do
66
+
67
+ let(:options) { double(fail_fast: true, stop_after: 3, detail: false) }
68
+ before do
69
+ subject.failure 'first'
70
+ subject.failure 'second'
71
+ end
72
+
73
+ it "returns true when the limit is reached " do
74
+ subject.failure 'third'
75
+ expect(subject.stop?).to be true
76
+ end
77
+
78
+ it "returns false when under the limit" do
79
+ expect(subject.stop?).to be false
80
+ end
81
+ end
82
+
83
+ context "with no failure limit" do
84
+ let(:options) { double(fail_fast: false, detail: false) }
85
+
86
+ it "return false" do
87
+ expect(subject.stop?).to be false
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,330 @@
1
+ require 'recog'
2
+ require 'yaml'
3
+
4
+
5
+ VALID_FILTER = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'util.os'}
6
+ NOMATCH_MATCH_KEY = {match_key: 'no_such_987', protocol: 'smb', database_type: 'util.os'}
7
+ NOMATCH_PROTO = {match_key: 'smb.native_os', protocol: 'no_such_987', database_type: 'util.os'}
8
+ NOMATCH_TYPE = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'no_such_987'}
9
+
10
+ describe Recog::Nizer do
11
+ subject { described_class }
12
+
13
+ describe ".match" do
14
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
15
+ data = line.strip
16
+ context "with smb_native_os:#{data}" do
17
+ let(:match_result) { subject.match('smb.native_os', data) }
18
+
19
+ it "returns a hash" do
20
+ expect(match_result.class).to eq(::Hash)
21
+ end
22
+
23
+ it "returns a successful match" do
24
+ expect(match_result['matched'].to_s).to match(/^[A-Z]/)
25
+ end
26
+
27
+ it "correctly matches service or os" do
28
+ if data =~ /^Windows/
29
+ expect(match_result['os.product']).to match(/^Windows/)
30
+ end
31
+ end
32
+
33
+ let(:nomatch_result) { subject.match('smb.native_os', 'no_such_987_76tgklh') }
34
+ it "returns a nil when data cannot be matched" do
35
+ expect(nomatch_result).to be_nil
36
+ end
37
+
38
+ let(:invalid_db_result) { subject.match('no_such_987', data) }
39
+ it "returns a nil when match_key search doesn't match" do
40
+ expect(invalid_db_result).to be_nil
41
+ end
42
+ end
43
+ end
44
+
45
+ line = 'non-existent'
46
+ context "with non-existent match" do
47
+ let(:match_result) {subject.match('smb.native_os', line) }
48
+ it "returns a nil" do
49
+ expect(match_result).to be_nil
50
+ end
51
+ end
52
+ end
53
+
54
+ describe ".match_all_db" do
55
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
56
+ data = line.strip
57
+ context "with smb_native_os:#{data}" do
58
+ let(:match_all_result) { subject.match_all_db(data, VALID_FILTER) }
59
+
60
+ it "returns an array" do
61
+ expect(match_all_result.class).to eq(::Array)
62
+ end
63
+
64
+ it "returns a successful match" do
65
+ expect(match_all_result[0]['matched']).to match(/^[A-Z]/)
66
+ end
67
+
68
+ it "correctly matches service or os" do
69
+ if data =~ /^Windows/
70
+ expect(match_all_result[0]['os.product']).to match(/^Windows/)
71
+ end
72
+ end
73
+
74
+ it "correctly matches protocol" do
75
+ expect(match_all_result[0]['service.protocol']).to eq('smb')
76
+ end
77
+
78
+ let(:no_filter_result) { subject.match_all_db(data) }
79
+ it "returns an array when searching without a filter" do
80
+ expect(no_filter_result.class).to eq(::Array)
81
+ end
82
+
83
+ it "returns a successful match when searching without a filter" do
84
+ expect(no_filter_result[0]['matched']).to match(/^[A-Z]/)
85
+ end
86
+
87
+ it "correctly matches service or os when searching without a filter" do
88
+ if data =~ /^Windows/
89
+ expect(no_filter_result[0]['os.product']).to match(/^Windows/)
90
+ end
91
+ end
92
+
93
+ let(:nomatch_db_result) { subject.match_all_db(data, NOMATCH_MATCH_KEY) }
94
+ it "returns an array when match_key search doesn't match" do
95
+ expect(nomatch_db_result.class).to eq(::Array)
96
+ end
97
+ it "returns an empty array when match_key search doesn't match" do
98
+ expect(nomatch_db_result).to be_empty
99
+ end
100
+
101
+ let(:nomatch_proto_result) { subject.match_all_db(data, NOMATCH_PROTO) }
102
+ it "returns an array when protocol search doesn't match" do
103
+ expect(nomatch_proto_result.class).to eq(::Array)
104
+ end
105
+ it "returns an empty array when protocol search doesn't match" do
106
+ expect(nomatch_proto_result).to be_empty
107
+ end
108
+
109
+ let(:nomatch_type_result) { subject.match_all_db(data, NOMATCH_TYPE) }
110
+ it "returns an array when database_type search doesn't match" do
111
+ expect(nomatch_type_result.class).to eq(::Array)
112
+ end
113
+ it "returns an empty array when database_type search doesn't match" do
114
+ expect(nomatch_proto_result).to be_empty
115
+ end
116
+ end
117
+ end
118
+
119
+ line = 'non-existent'
120
+ context "with non-existent match" do
121
+ let(:match_result) {subject.match_all_db(line) }
122
+ it "returns an array" do
123
+ expect(match_result.class).to eq(::Array)
124
+ end
125
+ it "returns an empty array" do
126
+ expect(match_result).to be_empty
127
+ end
128
+ end
129
+ end
130
+
131
+ describe ".multi_match" do
132
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
133
+ data = line.strip
134
+
135
+ context "with smb_native_os:#{data}" do
136
+ let(:match_results) {subject.multi_match('smb.native_os', data) }
137
+
138
+ it "returns an array" do
139
+ expect(match_results.class).to eq(::Array)
140
+ end
141
+
142
+ it "returns at least one successful match" do
143
+ expect(match_results.size).to be > 0
144
+ end
145
+
146
+ it "correctly matches service or os" do
147
+ match_results do |mr|
148
+ if data =~ /^Windows/
149
+ expect(mr['os.product']).to match(/^Windows/)
150
+ end
151
+ end
152
+ end
153
+
154
+ let(:invalid_db_result) { subject.multi_match('no_such_987', data) }
155
+ it "returns an array when passed an invalid match_key" do
156
+ expect(invalid_db_result.class).to eq(::Array)
157
+ end
158
+
159
+ it "returns an empty array when passed an invalid match_key" do
160
+ expect(invalid_db_result).to be_empty
161
+ end
162
+ end
163
+
164
+ end
165
+
166
+ data = 'Windows Server 2012 R2 Standard 9600'
167
+ context "with {data}" do
168
+ let(:match_results) {subject.multi_match('smb.native_os', data) }
169
+
170
+ it "returns an array" do
171
+ expect(match_results.class).to eq(::Array)
172
+ end
173
+
174
+ it "returns at least two successful matches" do
175
+ expect(match_results.size).to be > 1
176
+ end
177
+
178
+ it "correctly matches os.product for all matches" do
179
+ match_results do |mr|
180
+ if data =~ /^Windows/
181
+ expect(mr['os.product']).to match(/^Windows/)
182
+ end
183
+ end
184
+ end
185
+
186
+ it "correctly matches protocol for all matches" do
187
+ match_results do |mr|
188
+ if data =~ /^Windows/
189
+ expect(mr['service.protocol']).to eq('smb')
190
+ end
191
+ end
192
+ end
193
+ end
194
+
195
+ line = 'non-existent'
196
+ context "with non-existent match" do
197
+ let(:match_results) {subject.multi_match('smb.native_os', line) }
198
+
199
+ it "returns an array" do
200
+ expect(match_results.class).to eq(::Array)
201
+ end
202
+
203
+ it "returns an empty array" do
204
+ expect(match_results).to be_empty
205
+ end
206
+ end
207
+ end
208
+
209
+ describe ".best_os_match" do
210
+ # Demonstrates how this method picks up additional attributes from other members of the winning
211
+ # os.product match group and applies them to the result.
212
+ matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_1.yml'))))
213
+ context "with best_os_match_1.yml" do
214
+ let(:result) { subject.best_os_match(matches1) }
215
+
216
+ it "returns a hash" do
217
+ expect(result.class).to eq(::Hash)
218
+ end
219
+
220
+ it "matches Windows 2008" do
221
+ expect(result['os.product']).to eq('Windows 2008')
222
+ end
223
+
224
+ it "matches Microsoft" do
225
+ expect(result['os.vendor']).to eq('Microsoft')
226
+ end
227
+
228
+ it "matches English" do
229
+ expect(result['os.language']).to eq('English')
230
+ end
231
+
232
+ it "matches service pack 2" do
233
+ expect(result['os.version']).to eq('Service Pack 2')
234
+ end
235
+ end
236
+
237
+ # Demonstrates how additive os.certainty values allow a 1.0 certainty rule to be overridden
238
+ # by multiple lower certainty matches
239
+ matches2 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_2.yml'))))
240
+ context "with best_os_match_2.yml" do
241
+ let(:result) { subject.best_os_match(matches2) }
242
+
243
+ it "returns a hash" do
244
+ expect(result.class).to eq(::Hash)
245
+ end
246
+
247
+ it "matches Windows 2012" do
248
+ expect(result['os.product']).to eq('Windows 2012')
249
+ end
250
+
251
+ it "matches Microsoft" do
252
+ expect(result['os.vendor']).to eq('Microsoft')
253
+ end
254
+
255
+ it "matches Arabic" do
256
+ expect(result['os.language']).to eq('Arabic')
257
+ end
258
+
259
+ it "matches service pack 1" do
260
+ expect(result['os.version']).to eq('Service Pack 1')
261
+ end
262
+ end
263
+
264
+ end
265
+
266
+ describe ".best_service_match" do
267
+ # Demonstrates how this method picks up additional attributes from other members of the winning
268
+ # service.product match group and applies them to the result.
269
+ matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_service_match_1.yml'))))
270
+ context "with best_service_match_1.yml" do
271
+ let(:result) { subject.best_service_match(matches1) }
272
+
273
+ it "returns a hash" do
274
+ expect(result.class).to eq(::Hash)
275
+ end
276
+
277
+ it "matches IIS" do
278
+ expect(result['service.product']).to eq('IIS')
279
+ end
280
+
281
+ it "matches Microsoft" do
282
+ expect(result['service.vendor']).to eq('Microsoft')
283
+ end
284
+
285
+ it "matches English" do
286
+ expect(result['service.language']).to eq('English')
287
+ end
288
+
289
+ it "matches version 6.0" do
290
+ expect(result['service.version'].to_i).to eq(6.0)
291
+ end
292
+ end
293
+
294
+ end
295
+
296
+
297
+ describe '.load_db' do
298
+ file_path = File.expand_path(File.join('spec', 'data', 'test_fingerprints.xml'))
299
+ context "with #{file_path}" do
300
+ let(:fp_db) { subject.load_db(file_path) }
301
+ it "loads without error" do
302
+ expect(fp_db).to be true
303
+ subject.unload_db()
304
+ end
305
+ end
306
+
307
+ context "with no path specified" do
308
+ let(:fp_db) { subject.load_db }
309
+ it "loads without error" do
310
+ expect(fp_db).to be true
311
+ subject.unload_db()
312
+ end
313
+ end
314
+
315
+ context "with empty file path" do
316
+ it "raises an error" do
317
+ expect { subject.load_db('') }.to raise_error(Errno::ENOENT)
318
+ subject.unload_db()
319
+ end
320
+ end
321
+
322
+ context "with invalid file path" do
323
+ it "raises an error" do
324
+ expect { subject.load_db('no_such_987_file_path') }.to raise_error(Errno::ENOENT)
325
+ subject.unload_db()
326
+ end
327
+ end
328
+ end
329
+
330
+ end