recog-intrigue 2.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. checksums.yaml +7 -0
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +17 -0
  4. data/.github/ISSUE_TEMPLATE/fingerprint_request.md +27 -0
  5. data/.github/PULL_REQUEST_TEMPLATE +24 -0
  6. data/.gitignore +14 -0
  7. data/.rbenv-gemset +1 -0
  8. data/.rspec +3 -0
  9. data/.ruby-gemset +1 -0
  10. data/.ruby-version +1 -0
  11. data/.travis.yml +25 -0
  12. data/.yardopts +1 -0
  13. data/CONTRIBUTING.md +171 -0
  14. data/COPYING +23 -0
  15. data/Gemfile +10 -0
  16. data/LICENSE +7 -0
  17. data/README.md +85 -0
  18. data/Rakefile +22 -0
  19. data/bin/recog_export +81 -0
  20. data/bin/recog_match +55 -0
  21. data/bin/recog_standardize +118 -0
  22. data/bin/recog_verify +64 -0
  23. data/cpe-remap.yaml +134 -0
  24. data/features/data/failing_banners_fingerprints.xml +20 -0
  25. data/features/data/matching_banners_fingerprints.xml +23 -0
  26. data/features/data/multiple_banners_fingerprints.xml +32 -0
  27. data/features/data/no_tests.xml +3 -0
  28. data/features/data/sample_banner.txt +2 -0
  29. data/features/data/successful_tests.xml +18 -0
  30. data/features/data/tests_with_failures.xml +20 -0
  31. data/features/data/tests_with_warnings.xml +17 -0
  32. data/features/match.feature +36 -0
  33. data/features/support/aruba.rb +3 -0
  34. data/features/support/env.rb +6 -0
  35. data/features/verify.feature +48 -0
  36. data/identifiers/README.md +47 -0
  37. data/identifiers/os_architecture.txt +20 -0
  38. data/identifiers/os_device.txt +52 -0
  39. data/identifiers/os_family.txt +160 -0
  40. data/identifiers/os_product.txt +199 -0
  41. data/identifiers/service_family.txt +185 -0
  42. data/identifiers/service_product.txt +255 -0
  43. data/identifiers/software_class.txt +26 -0
  44. data/identifiers/software_family.txt +91 -0
  45. data/identifiers/software_product.txt +333 -0
  46. data/identifiers/vendor.txt +405 -0
  47. data/lib/recog.rb +4 -0
  48. data/lib/recog/db.rb +78 -0
  49. data/lib/recog/db_manager.rb +31 -0
  50. data/lib/recog/fingerprint.rb +280 -0
  51. data/lib/recog/fingerprint/regexp_factory.rb +56 -0
  52. data/lib/recog/fingerprint/test.rb +18 -0
  53. data/lib/recog/formatter.rb +51 -0
  54. data/lib/recog/match_reporter.rb +77 -0
  55. data/lib/recog/matcher.rb +94 -0
  56. data/lib/recog/matcher_factory.rb +14 -0
  57. data/lib/recog/nizer.rb +347 -0
  58. data/lib/recog/verifier.rb +39 -0
  59. data/lib/recog/verifier_factory.rb +13 -0
  60. data/lib/recog/verify_reporter.rb +86 -0
  61. data/lib/recog/version.rb +3 -0
  62. data/misc/convert_mysql_err +61 -0
  63. data/misc/order.xsl +17 -0
  64. data/recog-intrigue.gemspec +45 -0
  65. data/requirements.txt +2 -0
  66. data/spec/data/best_os_match_1.yml +17 -0
  67. data/spec/data/best_os_match_2.yml +17 -0
  68. data/spec/data/best_service_match_1.yml +17 -0
  69. data/spec/data/smb_native_os.txt +25 -0
  70. data/spec/data/test_fingerprints.xml +36 -0
  71. data/spec/data/verification_fingerprints.xml +86 -0
  72. data/spec/data/whitespaced_fingerprint.xml +5 -0
  73. data/spec/lib/fingerprint_self_test_spec.rb +174 -0
  74. data/spec/lib/recog/db_spec.rb +98 -0
  75. data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +73 -0
  76. data/spec/lib/recog/fingerprint_spec.rb +112 -0
  77. data/spec/lib/recog/formatter_spec.rb +69 -0
  78. data/spec/lib/recog/match_reporter_spec.rb +91 -0
  79. data/spec/lib/recog/nizer_spec.rb +330 -0
  80. data/spec/lib/recog/verify_reporter_spec.rb +113 -0
  81. data/spec/spec_helper.rb +82 -0
  82. data/update_cpes.py +186 -0
  83. data/xml/apache_modules.xml +1911 -0
  84. data/xml/apache_os.xml +273 -0
  85. data/xml/architecture.xml +36 -0
  86. data/xml/dns_versionbind.xml +761 -0
  87. data/xml/fingerprints.xsd +128 -0
  88. data/xml/ftp_banners.xml +1553 -0
  89. data/xml/h323_callresp.xml +603 -0
  90. data/xml/hp_pjl_id.xml +358 -0
  91. data/xml/html_title.xml +1630 -0
  92. data/xml/http_cookies.xml +411 -0
  93. data/xml/http_servers.xml +3195 -0
  94. data/xml/http_wwwauth.xml +595 -0
  95. data/xml/imap_banners.xml +245 -0
  96. data/xml/ldap_searchresult.xml +711 -0
  97. data/xml/mdns_device-info_txt.xml +1796 -0
  98. data/xml/mdns_workstation_txt.xml +15 -0
  99. data/xml/mysql_banners.xml +1649 -0
  100. data/xml/mysql_error.xml +871 -0
  101. data/xml/nntp_banners.xml +82 -0
  102. data/xml/ntp_banners.xml +1223 -0
  103. data/xml/operating_system.xml +629 -0
  104. data/xml/pop_banners.xml +499 -0
  105. data/xml/rsh_resp.xml +76 -0
  106. data/xml/rtsp_servers.xml +76 -0
  107. data/xml/sip_banners.xml +359 -0
  108. data/xml/sip_user_agents.xml +221 -0
  109. data/xml/smb_native_lm.xml +62 -0
  110. data/xml/smb_native_os.xml +662 -0
  111. data/xml/smtp_banners.xml +1690 -0
  112. data/xml/smtp_debug.xml +39 -0
  113. data/xml/smtp_ehlo.xml +49 -0
  114. data/xml/smtp_expn.xml +82 -0
  115. data/xml/smtp_help.xml +157 -0
  116. data/xml/smtp_mailfrom.xml +20 -0
  117. data/xml/smtp_noop.xml +44 -0
  118. data/xml/smtp_quit.xml +29 -0
  119. data/xml/smtp_rcptto.xml +25 -0
  120. data/xml/smtp_rset.xml +26 -0
  121. data/xml/smtp_turn.xml +26 -0
  122. data/xml/smtp_vrfy.xml +89 -0
  123. data/xml/snmp_sysdescr.xml +6507 -0
  124. data/xml/snmp_sysobjid.xml +430 -0
  125. data/xml/ssh_banners.xml +1968 -0
  126. data/xml/telnet_banners.xml +1595 -0
  127. data/xml/x11_banners.xml +232 -0
  128. data/xml/x509_issuers.xml +134 -0
  129. data/xml/x509_subjects.xml +1268 -0
  130. metadata +304 -0
@@ -0,0 +1,69 @@
1
+ require 'recog/formatter'
2
+
3
+ describe Recog::Formatter do
4
+ let(:output) { StringIO.new }
5
+
6
+ context "with no color" do
7
+ subject { Recog::Formatter.new(double(color: false), output) }
8
+
9
+ describe "#message" do
10
+ it "outputs the text" do
11
+ subject.status_message 'some text'
12
+ expect(output.string).to eq("some text\n")
13
+ end
14
+ end
15
+
16
+ describe "#success_message" do
17
+ it "outputs the text" do
18
+ subject.success_message 'a success'
19
+ expect(output.string).to eq("a success\n")
20
+ end
21
+ end
22
+
23
+ describe "#warning_message" do
24
+ it "outputs the text" do
25
+ subject.warning_message 'a warning'
26
+ expect(output.string).to eq("a warning\n")
27
+ end
28
+ end
29
+
30
+ describe "#failure_message" do
31
+ it "outputs the text" do
32
+ subject.failure_message 'a failure'
33
+ expect(output.string).to eq("a failure\n")
34
+ end
35
+ end
36
+ end
37
+
38
+ context "with color" do
39
+ subject { Recog::Formatter.new(double(color: true), output) }
40
+
41
+ describe "#message" do
42
+ it "outputs the text in white" do
43
+ subject.status_message 'some text'
44
+ expect(output.string).to eq("\e[15msome text\e[0m\n")
45
+ end
46
+ end
47
+
48
+ describe "#success_message" do
49
+ it "outputs the text in green" do
50
+ subject.success_message 'a success'
51
+ expect(output.string).to eq("\e[32ma success\e[0m\n")
52
+ end
53
+ end
54
+
55
+ describe "#warning_message" do
56
+ it "outputs the text in yellow" do
57
+ subject.warning_message 'a warning'
58
+ expect(output.string).to eq("\e[33ma warning\e[0m\n")
59
+ end
60
+ end
61
+
62
+ describe "#failure_message" do
63
+ it "outputs the text in red" do
64
+ subject.failure_message 'a failure'
65
+ expect(output.string).to eq("\e[31ma failure\e[0m\n")
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,91 @@
1
+ require 'recog/match_reporter'
2
+
3
+ describe Recog::MatchReporter do
4
+ let(:options) { double(detail: false, quiet: false) }
5
+ let(:formatter) { double('formatter').as_null_object }
6
+ subject { Recog::MatchReporter.new(options, formatter) }
7
+
8
+ def run_report
9
+ subject.report do
10
+ subject.increment_line_count
11
+ subject.match 'a match'
12
+ subject.failure 'a failure'
13
+ end
14
+ end
15
+
16
+ describe "#report" do
17
+ it "prints matches" do
18
+ expect(formatter).to receive(:success_message).with('a match')
19
+ run_report
20
+ end
21
+
22
+ it "prints failures" do
23
+ expect(formatter).to receive(:failure_message).with('a failure')
24
+ run_report
25
+ end
26
+
27
+ context "with detail" do
28
+ subject { Recog::MatchReporter.new(double(detail: true, quiet: false), formatter) }
29
+
30
+ it "prints the lines processed" do
31
+ expect(formatter).to receive(:status_message).with("\nProcessed 1 lines")
32
+ run_report
33
+ end
34
+
35
+ it "prints summary" do
36
+ expect(formatter).to receive(:failure_message).with("SUMMARY: 1 matches and 1 failures")
37
+ run_report
38
+ end
39
+ end
40
+ end
41
+
42
+ describe "#print_summary" do
43
+ context "with all matches" do
44
+ before { subject.match 'match' }
45
+
46
+ it "prints a successful summary" do
47
+ msg = "SUMMARY: 1 matches and 0 failures"
48
+ expect(formatter).to receive(:success_message).with(msg)
49
+ subject.print_summary
50
+ end
51
+ end
52
+
53
+ context "with failures" do
54
+ before { subject.failure 'fail' }
55
+
56
+ it "prints a failure summary" do
57
+ msg = "SUMMARY: 0 matches and 1 failures"
58
+ expect(formatter).to receive(:failure_message).with(msg)
59
+ subject.print_summary
60
+ end
61
+ end
62
+ end
63
+
64
+ describe "#stop?" do
65
+ context "with a failure limit" do
66
+
67
+ let(:options) { double(fail_fast: true, stop_after: 3, detail: false) }
68
+ before do
69
+ subject.failure 'first'
70
+ subject.failure 'second'
71
+ end
72
+
73
+ it "returns true when the limit is reached " do
74
+ subject.failure 'third'
75
+ expect(subject.stop?).to be true
76
+ end
77
+
78
+ it "returns false when under the limit" do
79
+ expect(subject.stop?).to be false
80
+ end
81
+ end
82
+
83
+ context "with no failure limit" do
84
+ let(:options) { double(fail_fast: false, detail: false) }
85
+
86
+ it "return false" do
87
+ expect(subject.stop?).to be false
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,330 @@
1
+ require 'recog'
2
+ require 'yaml'
3
+
4
+
5
+ VALID_FILTER = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'util.os'}
6
+ NOMATCH_MATCH_KEY = {match_key: 'no_such_987', protocol: 'smb', database_type: 'util.os'}
7
+ NOMATCH_PROTO = {match_key: 'smb.native_os', protocol: 'no_such_987', database_type: 'util.os'}
8
+ NOMATCH_TYPE = {match_key: 'smb.native_os', protocol: 'smb', database_type: 'no_such_987'}
9
+
10
+ describe Recog::Nizer do
11
+ subject { described_class }
12
+
13
+ describe ".match" do
14
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
15
+ data = line.strip
16
+ context "with smb_native_os:#{data}" do
17
+ let(:match_result) { subject.match('smb.native_os', data) }
18
+
19
+ it "returns a hash" do
20
+ expect(match_result.class).to eq(::Hash)
21
+ end
22
+
23
+ it "returns a successful match" do
24
+ expect(match_result['matched'].to_s).to match(/^[A-Z]/)
25
+ end
26
+
27
+ it "correctly matches service or os" do
28
+ if data =~ /^Windows/
29
+ expect(match_result['os.product']).to match(/^Windows/)
30
+ end
31
+ end
32
+
33
+ let(:nomatch_result) { subject.match('smb.native_os', 'no_such_987_76tgklh') }
34
+ it "returns a nil when data cannot be matched" do
35
+ expect(nomatch_result).to be_nil
36
+ end
37
+
38
+ let(:invalid_db_result) { subject.match('no_such_987', data) }
39
+ it "returns a nil when match_key search doesn't match" do
40
+ expect(invalid_db_result).to be_nil
41
+ end
42
+ end
43
+ end
44
+
45
+ line = 'non-existent'
46
+ context "with non-existent match" do
47
+ let(:match_result) {subject.match('smb.native_os', line) }
48
+ it "returns a nil" do
49
+ expect(match_result).to be_nil
50
+ end
51
+ end
52
+ end
53
+
54
+ describe ".match_all_db" do
55
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
56
+ data = line.strip
57
+ context "with smb_native_os:#{data}" do
58
+ let(:match_all_result) { subject.match_all_db(data, VALID_FILTER) }
59
+
60
+ it "returns an array" do
61
+ expect(match_all_result.class).to eq(::Array)
62
+ end
63
+
64
+ it "returns a successful match" do
65
+ expect(match_all_result[0]['matched']).to match(/^[A-Z]/)
66
+ end
67
+
68
+ it "correctly matches service or os" do
69
+ if data =~ /^Windows/
70
+ expect(match_all_result[0]['os.product']).to match(/^Windows/)
71
+ end
72
+ end
73
+
74
+ it "correctly matches protocol" do
75
+ expect(match_all_result[0]['service.protocol']).to eq('smb')
76
+ end
77
+
78
+ let(:no_filter_result) { subject.match_all_db(data) }
79
+ it "returns an array when searching without a filter" do
80
+ expect(no_filter_result.class).to eq(::Array)
81
+ end
82
+
83
+ it "returns a successful match when searching without a filter" do
84
+ expect(no_filter_result[0]['matched']).to match(/^[A-Z]/)
85
+ end
86
+
87
+ it "correctly matches service or os when searching without a filter" do
88
+ if data =~ /^Windows/
89
+ expect(no_filter_result[0]['os.product']).to match(/^Windows/)
90
+ end
91
+ end
92
+
93
+ let(:nomatch_db_result) { subject.match_all_db(data, NOMATCH_MATCH_KEY) }
94
+ it "returns an array when match_key search doesn't match" do
95
+ expect(nomatch_db_result.class).to eq(::Array)
96
+ end
97
+ it "returns an empty array when match_key search doesn't match" do
98
+ expect(nomatch_db_result).to be_empty
99
+ end
100
+
101
+ let(:nomatch_proto_result) { subject.match_all_db(data, NOMATCH_PROTO) }
102
+ it "returns an array when protocol search doesn't match" do
103
+ expect(nomatch_proto_result.class).to eq(::Array)
104
+ end
105
+ it "returns an empty array when protocol search doesn't match" do
106
+ expect(nomatch_proto_result).to be_empty
107
+ end
108
+
109
+ let(:nomatch_type_result) { subject.match_all_db(data, NOMATCH_TYPE) }
110
+ it "returns an array when database_type search doesn't match" do
111
+ expect(nomatch_type_result.class).to eq(::Array)
112
+ end
113
+ it "returns an empty array when database_type search doesn't match" do
114
+ expect(nomatch_proto_result).to be_empty
115
+ end
116
+ end
117
+ end
118
+
119
+ line = 'non-existent'
120
+ context "with non-existent match" do
121
+ let(:match_result) {subject.match_all_db(line) }
122
+ it "returns an array" do
123
+ expect(match_result.class).to eq(::Array)
124
+ end
125
+ it "returns an empty array" do
126
+ expect(match_result).to be_empty
127
+ end
128
+ end
129
+ end
130
+
131
+ describe ".multi_match" do
132
+ File.readlines(File.expand_path(File.join('spec', 'data', 'smb_native_os.txt'))).each do |line|
133
+ data = line.strip
134
+
135
+ context "with smb_native_os:#{data}" do
136
+ let(:match_results) {subject.multi_match('smb.native_os', data) }
137
+
138
+ it "returns an array" do
139
+ expect(match_results.class).to eq(::Array)
140
+ end
141
+
142
+ it "returns at least one successful match" do
143
+ expect(match_results.size).to be > 0
144
+ end
145
+
146
+ it "correctly matches service or os" do
147
+ match_results do |mr|
148
+ if data =~ /^Windows/
149
+ expect(mr['os.product']).to match(/^Windows/)
150
+ end
151
+ end
152
+ end
153
+
154
+ let(:invalid_db_result) { subject.multi_match('no_such_987', data) }
155
+ it "returns an array when passed an invalid match_key" do
156
+ expect(invalid_db_result.class).to eq(::Array)
157
+ end
158
+
159
+ it "returns an empty array when passed an invalid match_key" do
160
+ expect(invalid_db_result).to be_empty
161
+ end
162
+ end
163
+
164
+ end
165
+
166
+ data = 'Windows Server 2012 R2 Standard 9600'
167
+ context "with {data}" do
168
+ let(:match_results) {subject.multi_match('smb.native_os', data) }
169
+
170
+ it "returns an array" do
171
+ expect(match_results.class).to eq(::Array)
172
+ end
173
+
174
+ it "returns at least two successful matches" do
175
+ expect(match_results.size).to be > 1
176
+ end
177
+
178
+ it "correctly matches os.product for all matches" do
179
+ match_results do |mr|
180
+ if data =~ /^Windows/
181
+ expect(mr['os.product']).to match(/^Windows/)
182
+ end
183
+ end
184
+ end
185
+
186
+ it "correctly matches protocol for all matches" do
187
+ match_results do |mr|
188
+ if data =~ /^Windows/
189
+ expect(mr['service.protocol']).to eq('smb')
190
+ end
191
+ end
192
+ end
193
+ end
194
+
195
+ line = 'non-existent'
196
+ context "with non-existent match" do
197
+ let(:match_results) {subject.multi_match('smb.native_os', line) }
198
+
199
+ it "returns an array" do
200
+ expect(match_results.class).to eq(::Array)
201
+ end
202
+
203
+ it "returns an empty array" do
204
+ expect(match_results).to be_empty
205
+ end
206
+ end
207
+ end
208
+
209
+ describe ".best_os_match" do
210
+ # Demonstrates how this method picks up additional attributes from other members of the winning
211
+ # os.product match group and applies them to the result.
212
+ matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_1.yml'))))
213
+ context "with best_os_match_1.yml" do
214
+ let(:result) { subject.best_os_match(matches1) }
215
+
216
+ it "returns a hash" do
217
+ expect(result.class).to eq(::Hash)
218
+ end
219
+
220
+ it "matches Windows 2008" do
221
+ expect(result['os.product']).to eq('Windows 2008')
222
+ end
223
+
224
+ it "matches Microsoft" do
225
+ expect(result['os.vendor']).to eq('Microsoft')
226
+ end
227
+
228
+ it "matches English" do
229
+ expect(result['os.language']).to eq('English')
230
+ end
231
+
232
+ it "matches service pack 2" do
233
+ expect(result['os.version']).to eq('Service Pack 2')
234
+ end
235
+ end
236
+
237
+ # Demonstrates how additive os.certainty values allow a 1.0 certainty rule to be overridden
238
+ # by multiple lower certainty matches
239
+ matches2 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_os_match_2.yml'))))
240
+ context "with best_os_match_2.yml" do
241
+ let(:result) { subject.best_os_match(matches2) }
242
+
243
+ it "returns a hash" do
244
+ expect(result.class).to eq(::Hash)
245
+ end
246
+
247
+ it "matches Windows 2012" do
248
+ expect(result['os.product']).to eq('Windows 2012')
249
+ end
250
+
251
+ it "matches Microsoft" do
252
+ expect(result['os.vendor']).to eq('Microsoft')
253
+ end
254
+
255
+ it "matches Arabic" do
256
+ expect(result['os.language']).to eq('Arabic')
257
+ end
258
+
259
+ it "matches service pack 1" do
260
+ expect(result['os.version']).to eq('Service Pack 1')
261
+ end
262
+ end
263
+
264
+ end
265
+
266
+ describe ".best_service_match" do
267
+ # Demonstrates how this method picks up additional attributes from other members of the winning
268
+ # service.product match group and applies them to the result.
269
+ matches1 = YAML.load(File.read(File.expand_path(File.join('spec', 'data', 'best_service_match_1.yml'))))
270
+ context "with best_service_match_1.yml" do
271
+ let(:result) { subject.best_service_match(matches1) }
272
+
273
+ it "returns a hash" do
274
+ expect(result.class).to eq(::Hash)
275
+ end
276
+
277
+ it "matches IIS" do
278
+ expect(result['service.product']).to eq('IIS')
279
+ end
280
+
281
+ it "matches Microsoft" do
282
+ expect(result['service.vendor']).to eq('Microsoft')
283
+ end
284
+
285
+ it "matches English" do
286
+ expect(result['service.language']).to eq('English')
287
+ end
288
+
289
+ it "matches version 6.0" do
290
+ expect(result['service.version'].to_i).to eq(6.0)
291
+ end
292
+ end
293
+
294
+ end
295
+
296
+
297
+ describe '.load_db' do
298
+ file_path = File.expand_path(File.join('spec', 'data', 'test_fingerprints.xml'))
299
+ context "with #{file_path}" do
300
+ let(:fp_db) { subject.load_db(file_path) }
301
+ it "loads without error" do
302
+ expect(fp_db).to be true
303
+ subject.unload_db()
304
+ end
305
+ end
306
+
307
+ context "with no path specified" do
308
+ let(:fp_db) { subject.load_db }
309
+ it "loads without error" do
310
+ expect(fp_db).to be true
311
+ subject.unload_db()
312
+ end
313
+ end
314
+
315
+ context "with empty file path" do
316
+ it "raises an error" do
317
+ expect { subject.load_db('') }.to raise_error(Errno::ENOENT)
318
+ subject.unload_db()
319
+ end
320
+ end
321
+
322
+ context "with invalid file path" do
323
+ it "raises an error" do
324
+ expect { subject.load_db('no_such_987_file_path') }.to raise_error(Errno::ENOENT)
325
+ subject.unload_db()
326
+ end
327
+ end
328
+ end
329
+
330
+ end