recog 2.3.7 → 2.3.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +9 -2
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +2 -4
- data/CONTRIBUTING.md +136 -37
- data/Gemfile +2 -5
- data/README.md +18 -16
- data/bin/recog_cleanup +16 -0
- data/bin/recog_standardize +142 -0
- data/cpe-remap.yaml +36 -1
- data/features/match.feature +4 -0
- data/features/support/aruba.rb +3 -0
- data/features/verify.feature +5 -0
- data/identifiers/README.md +56 -0
- data/identifiers/hw_device.txt +77 -0
- data/identifiers/hw_family.txt +96 -0
- data/identifiers/hw_product.txt +328 -0
- data/identifiers/os_architecture.txt +20 -0
- data/identifiers/os_device.txt +94 -0
- data/identifiers/os_family.txt +325 -0
- data/identifiers/os_product.txt +420 -0
- data/identifiers/service_family.txt +272 -0
- data/identifiers/service_product.txt +557 -0
- data/identifiers/software_class.txt +26 -0
- data/identifiers/software_family.txt +91 -0
- data/identifiers/software_product.txt +333 -0
- data/identifiers/vendor.txt +891 -0
- data/lib/recog/version.rb +1 -1
- data/requirements.txt +1 -1
- data/spec/lib/fingerprint_self_test_spec.rb +1 -1
- data/spec/lib/recog/fingerprint/regexp_factory_spec.rb +1 -1
- data/update_cpes.py +4 -1
- data/xml/apache_modules.xml +292 -5
- data/xml/apache_os.xml +50 -2
- data/xml/architecture.xml +19 -7
- data/xml/dns_versionbind.xml +200 -26
- data/xml/favicons.xml +1701 -0
- data/xml/ftp_banners.xml +276 -16
- data/xml/h323_callresp.xml +112 -12
- data/xml/hp_pjl_id.xml +47 -5
- data/xml/html_title.xml +1419 -72
- data/xml/http_cookies.xml +77 -10
- data/xml/http_servers.xml +898 -47
- data/xml/http_wwwauth.xml +154 -27
- data/xml/imap_banners.xml +23 -13
- data/xml/ldap_searchresult.xml +81 -9
- data/xml/mdns_device-info_txt.xml +194 -17
- data/xml/mdns_workstation_txt.xml +4 -2
- data/xml/mysql_banners.xml +554 -45
- data/xml/mysql_error.xml +113 -6
- data/xml/nntp_banners.xml +10 -2
- data/xml/ntp_banners.xml +95 -11
- data/xml/operating_system.xml +90 -3
- data/xml/pop_banners.xml +32 -31
- data/xml/rsh_resp.xml +11 -2
- data/xml/rtsp_servers.xml +43 -23
- data/xml/sip_banners.xml +9 -14
- data/xml/sip_user_agents.xml +69 -3
- data/xml/smb_native_lm.xml +10 -2
- data/xml/smb_native_os.xml +80 -2
- data/xml/smtp_banners.xml +233 -13
- data/xml/smtp_debug.xml +6 -4
- data/xml/smtp_ehlo.xml +7 -5
- data/xml/smtp_expn.xml +13 -4
- data/xml/smtp_help.xml +23 -4
- data/xml/smtp_mailfrom.xml +5 -2
- data/xml/smtp_noop.xml +6 -5
- data/xml/smtp_quit.xml +5 -4
- data/xml/smtp_rcptto.xml +5 -2
- data/xml/smtp_rset.xml +4 -4
- data/xml/smtp_turn.xml +4 -4
- data/xml/smtp_vrfy.xml +14 -4
- data/xml/snmp_sysdescr.xml +776 -52
- data/xml/snmp_sysobjid.xml +47 -2
- data/xml/ssh_banners.xml +259 -80
- data/xml/telnet_banners.xml +376 -23
- data/xml/x11_banners.xml +27 -4
- data/xml/x509_issuers.xml +37 -13
- data/xml/x509_subjects.xml +525 -55
- metadata +29 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 477c1b984da6788f9acbe0f8868fe07b500c0a6c9b94691a23535befba43b912
|
4
|
+
data.tar.gz: a23f96a8ae8212ba1521455ccbfbe60f355706eb04fbe309768cd538a5ff90a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b758ee85f8fb433fb2d0ef8ad07e627bb941ad86d5d3f07ebb768ecda8c8a00521e37c0dff81e5e2c3b7065584b7d4bef283fb6ceea0da45c952e57301879d2c
|
7
|
+
data.tar.gz: d07e8d168442426db1cee39ecaf6d2f7c337b20bf41799115354099185daa2e4696c58df2fddba9d025f05386fd426ce4c15bbc06eea04e47dff01642a86cd95
|
data/.gitignore
CHANGED
@@ -1,6 +1,4 @@
|
|
1
1
|
# Ruby and tooling specific
|
2
|
-
.ruby-version
|
3
|
-
.ruby-gemset
|
4
2
|
.yardoc
|
5
3
|
coverage/
|
6
4
|
doc/
|
@@ -8,9 +6,18 @@ pkg/
|
|
8
6
|
|
9
7
|
/Gemfile.lock
|
10
8
|
|
9
|
+
#Python specific
|
10
|
+
venv
|
11
|
+
|
11
12
|
# IDE specific
|
12
13
|
.vscode/
|
13
14
|
.idea
|
14
15
|
|
15
16
|
# Misc
|
16
17
|
**/.DS_Store
|
18
|
+
|
19
|
+
# CPE XML
|
20
|
+
official-cpe-dictionary*.xml
|
21
|
+
|
22
|
+
# CPE Remap Errors
|
23
|
+
errors.txt
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
recog
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.6.6
|
data/.travis.yml
CHANGED
data/CONTRIBUTING.md
CHANGED
@@ -3,9 +3,24 @@
|
|
3
3
|
The users and maintainers of Recog would greatly appreciate any contributions
|
4
4
|
you can make to the project. These contributions typically come in the form of
|
5
5
|
filed bugs/issues or pull requests (PRs). These contributions routinely result
|
6
|
-
in new versions of the [recog gem](https://rubygems.org/gems/recog)
|
6
|
+
in new versions of the [recog gem](https://rubygems.org/gems/recog) being
|
7
7
|
released. The process for everything is described below.
|
8
8
|
|
9
|
+
## Table of Contents
|
10
|
+
|
11
|
+
1. [Contributing Issues / Bug Reports](#contributing-issues-/-bug-reports)
|
12
|
+
1. [Contributing Code](#contributing-code)
|
13
|
+
1. [Fork and Clone](#fork-and-clone)
|
14
|
+
1. [Branch and Improve](#branch-and-improve)
|
15
|
+
1. [Testing](#testing)
|
16
|
+
1. [Fingerprints](#fingerprints)
|
17
|
+
1. [Best Practices](#best-practices)
|
18
|
+
1. [Fingerprint Testing](#fingerprint-testing)
|
19
|
+
1. [Updating CPEs](#updating-cpes)
|
20
|
+
1. [Project Operations](#project-operations)
|
21
|
+
1. [Landing PRs](#landing-prs)
|
22
|
+
1. [Releasing New Versions](#releasing-new-versions)
|
23
|
+
|
9
24
|
## Contributing Issues / Bug Reports
|
10
25
|
|
11
26
|
If you encounter any bugs or problems with Recog, please file them
|
@@ -14,6 +29,8 @@ possible. If the bug is straight-forward enough and you understand the fix for
|
|
14
29
|
the bug well enough, you may take the simpler, less-paperwork route and simply
|
15
30
|
fill a PR with the fix and the necessary details.
|
16
31
|
|
32
|
+
[^back to top](#contributing-to-recog)
|
33
|
+
|
17
34
|
## Contributing Code
|
18
35
|
|
19
36
|
Recog uses a model nearly identical to that of
|
@@ -26,33 +43,39 @@ this document.
|
|
26
43
|
|
27
44
|
On the other hand, if you haven't, read on!
|
28
45
|
|
46
|
+
[^back to top](#contributing-to-recog)
|
47
|
+
|
29
48
|
### Fork and Clone
|
30
49
|
|
31
50
|
Generally, this should only need to be done once, or if you need to start over.
|
32
51
|
|
33
52
|
1. Fork Recog: Visit https://github.com/rapid7/recog and click Fork,
|
34
53
|
selecting your github account if prompted
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
```
|
40
|
-
git remote add upstream git://github.com/rapid7/recog.git
|
41
|
-
```
|
42
|
-
4. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream. It should look something like the following, in particular the second `fetch` option:
|
54
|
+
1. Clone `git@github.com:<your-github-username>/recog.git`, replacing
|
55
|
+
`<your-github-username>` with, you guessed it, your Github username.
|
56
|
+
1. Add the master Recog repository as your upstream:
|
43
57
|
|
58
|
+
```bash
|
59
|
+
git remote add upstream git://github.com/rapid7/recog.git
|
44
60
|
```
|
61
|
+
|
62
|
+
1. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream. It should look something like the following, in particular the second `fetch` option:
|
63
|
+
|
64
|
+
```bash
|
45
65
|
[remote "upstream"]
|
46
66
|
url = git@github.com:rapid7/recog.git
|
47
67
|
fetch = +refs/heads/*:refs/remotes/upstream/*
|
48
68
|
fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
|
49
69
|
```
|
50
|
-
5. Fetch the latest revisions, including PRs:
|
51
70
|
|
52
|
-
|
71
|
+
1. Fetch the latest revisions, including PRs:
|
72
|
+
|
73
|
+
```bash
|
53
74
|
git fetch --all
|
54
75
|
```
|
55
76
|
|
77
|
+
[^back to top](#contributing-to-recog)
|
78
|
+
|
56
79
|
### Branch and Improve
|
57
80
|
|
58
81
|
If you have a contribution to make, first create a branch to contain your
|
@@ -60,7 +83,7 @@ work. The name is yours to choose, however generally it should roughly
|
|
60
83
|
describe what you are doing. In this example, and from here on out, the
|
61
84
|
branch will be FOO, but you should obviously change this:
|
62
85
|
|
63
|
-
```
|
86
|
+
```bash
|
64
87
|
git fetch --all
|
65
88
|
git checkout master
|
66
89
|
git rebase upstream/master
|
@@ -73,17 +96,66 @@ Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/rec
|
|
73
96
|
|
74
97
|
Now push your changes to your fork:
|
75
98
|
|
76
|
-
```
|
99
|
+
```bash
|
77
100
|
git push origin FOO
|
78
101
|
```
|
79
102
|
|
80
103
|
Finally, submit the PR. Navigate to ```https://github.com/<your-github-username>/recog/compare/FOO```, fill in the details and submit.
|
81
104
|
|
105
|
+
[^back to top](#contributing-to-recog)
|
106
|
+
|
82
107
|
### Testing
|
83
108
|
|
84
109
|
When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR. To do this, simply run `rake tests`.
|
85
110
|
|
86
|
-
|
111
|
+
[^back to top](#contributing-to-recog)
|
112
|
+
|
113
|
+
## Fingerprints
|
114
|
+
|
115
|
+
### Best Practices
|
116
|
+
|
117
|
+
* Create a single fingerprint for each product as long as the pattern remains clear and readable. If that is not possible, the pattern should be logically decomposed into additional fingerprints.
|
118
|
+
|
119
|
+
* Create regular expressions that allow for flexible version number matching. This ensures greater probability of matching a product. For example, all known public releases of a product report either `major.minor` or `major.minor.build` format version numbers. If the fingerprint strictly matches this version number format, it would fail to match a modified build of the product that reports only a `major` version number format.
|
120
|
+
|
121
|
+
[^back to top](#contributing-to-recog)
|
122
|
+
|
123
|
+
### Fingerprint Testing
|
124
|
+
|
125
|
+
Once a fingerprint has been added, the `example` entries can be tested by executing `bin/recog_verify` against the fingerprint file:
|
126
|
+
|
127
|
+
```shell
|
128
|
+
bin/recog_verify xml/ssh_banners.xml
|
129
|
+
```
|
130
|
+
|
131
|
+
Matches can be tested on the command-line in a similar fashion:
|
132
|
+
|
133
|
+
```shell
|
134
|
+
$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
|
135
|
+
MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
136
|
+
```
|
137
|
+
|
138
|
+
[^back to top](#contributing-to-recog)
|
139
|
+
|
140
|
+
|
141
|
+
### Standardizing Vendors, Products, and Services
|
142
|
+
|
143
|
+
Given the number of fingerprints in Recog, it can be common for specific products, vendors, or services to be identified with different spellings and casing.
|
144
|
+
To limit the creep of slightly-different-names, the `bin/recog_standardize` script can be used to extract all identifiers and merge them into the known lists.
|
145
|
+
|
146
|
+
To get started, run the `recog_standardize` tool:
|
147
|
+
```shell
|
148
|
+
ruby bin/recog_standardize
|
149
|
+
```
|
150
|
+
|
151
|
+
Review any new additions to the text files under `identifiers/`. If any of these names are close to an existing name, update the offending fingerprint to use
|
152
|
+
the existing name instead. Once the fingerprints are fixed, removed the "extra" names from the identifiers files, and run the tool again.
|
153
|
+
|
154
|
+
|
155
|
+
[^back to top](#contributing-to-recog)
|
156
|
+
|
157
|
+
|
158
|
+
### Updating CPEs
|
87
159
|
|
88
160
|
There exists some automation to update the CPEs that might be asserted with
|
89
161
|
some recog fingerprints. This should be run periodically to ensure that all
|
@@ -91,30 +163,47 @@ fingerprints that could have CPEs do, etc.
|
|
91
163
|
|
92
164
|
First, setup a python3 venv:
|
93
165
|
|
94
|
-
```
|
166
|
+
```bash
|
95
167
|
python3 -m venv venv
|
96
|
-
source venv/bin/activate
|
168
|
+
source venv/{bin,Scripts}/activate
|
97
169
|
pip install -r requirements.txt
|
98
170
|
```
|
99
171
|
|
100
172
|
Download the latest CPE 2.3 dictionary:
|
101
173
|
|
102
|
-
|
103
|
-
|
104
|
-
|
174
|
+
```bash
|
175
|
+
curl -o official-cpe-dictionary_v2.3.xml.gz https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz && \
|
176
|
+
gunzip official-cpe-dictionary_v2.3.xml.gz
|
177
|
+
```
|
105
178
|
|
106
|
-
Run the CPE automation against every XML file
|
179
|
+
Run the CPE automation against every XML file:
|
107
180
|
|
108
|
-
|
109
|
-
|
110
|
-
|
181
|
+
```bash
|
182
|
+
# Update the CPEs (sequentially)
|
183
|
+
ls xml/*.xml | xargs -i python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml 2>>errors.txt
|
184
|
+
```
|
185
|
+
|
186
|
+
You may want to use GNU `parallel` to speed things up:
|
187
|
+
```bash
|
188
|
+
# Update the CPEs (with GNU Parallel)
|
189
|
+
ls xml/*.xml | parallel --gnu "python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml" 2>>errors.txt
|
190
|
+
```
|
191
|
+
|
192
|
+
Clean up the whitespace across all fingerprints:
|
193
|
+
```bash
|
194
|
+
ruby bin/recog_cleanup
|
195
|
+
```
|
111
196
|
|
112
197
|
Any mismatched fingerprints will be listed in `errors.txt` for eventual
|
113
198
|
maintenance. The `cpe-remap.yaml` file can be used to map between
|
114
199
|
vendor/product/etc differences between Recog and CPE, or to work around bugs in
|
115
200
|
either.
|
116
201
|
|
117
|
-
|
202
|
+
[^back to top](#contributing-to-recog)
|
203
|
+
|
204
|
+
## Project Operations
|
205
|
+
|
206
|
+
### Landing PRs
|
118
207
|
|
119
208
|
(Note: this portion is a work-in-progress. Please update it as things change)
|
120
209
|
|
@@ -126,46 +215,56 @@ In short:
|
|
126
215
|
1. Follow the "Fork and Clone" steps from above
|
127
216
|
2. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream. It should look something like the following, in particular the second `fetch` option:
|
128
217
|
|
129
|
-
```
|
218
|
+
```bash
|
130
219
|
[remote "upstream"]
|
131
220
|
url = git@github.com:rapid7/recog.git
|
132
221
|
fetch = +refs/heads/*:refs/remotes/upstream/*
|
133
222
|
fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
|
134
223
|
```
|
224
|
+
|
135
225
|
3. Fetch the latest revisions, including PRs:
|
136
226
|
|
137
|
-
```
|
227
|
+
```bash
|
138
228
|
git fetch --all
|
139
229
|
```
|
230
|
+
|
140
231
|
4. Checkout and branch the PR for testing. Replace ```PR``` below with the actual PR # in question:
|
141
232
|
|
142
|
-
```
|
233
|
+
```bash
|
143
234
|
git checkout -b landing-PR upstream/pr/PR
|
144
235
|
```
|
236
|
+
|
145
237
|
5. Test the PR (see the Testing section above)
|
146
238
|
6. Merge with master, re-test, validate and push:
|
147
239
|
|
148
|
-
```
|
240
|
+
```bash
|
149
241
|
git checkout -b upstream-master --track upstream/master
|
150
242
|
git merge -S --no-ff --edit landing-PR # merge the PR into upstream-master
|
243
|
+
|
151
244
|
# re-test if/as necessary
|
152
245
|
git push upstream upstream-master:master --dry-run # confirm you are pushing what you expect
|
246
|
+
|
153
247
|
git push upstream upstream-master:master # push upstream-master to upstream:master
|
154
248
|
```
|
249
|
+
|
155
250
|
7. If applicable, release a new version (see next section)
|
156
251
|
|
157
|
-
|
252
|
+
[^back to top](#contributing-to-recog)
|
253
|
+
|
254
|
+
### Releasing New Versions
|
158
255
|
|
159
256
|
When Recog's critical parts are modified, for example its fingerprints or underlying supporting code, a new version _must_ eventually be released. These new releases can then be optionally included in projects such as Metasploit or products such as Rapid7's Nexpose in a controlled manner. Releases for non-functional updates such as updates to documentation are not necessary.
|
160
257
|
|
161
258
|
When a new version of Recog is to be released, you _must_ follow the instructions below.
|
162
259
|
|
163
260
|
1. If are not already a Recog project contributor for the Recog gem (you'd be listed [here under OWNERS](https://rubygems.org/gems/recog)), become one:
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
261
|
+
1. Get an account on [Rubygems](https://rubygems.org)
|
262
|
+
1. Contact one of the Recog project contributors (listed [here under OWNERS](https://rubygems.org/gems/recog) and have them add you to the Recog gem. They'll need to run: `gem owner recog -a EMAIL`
|
263
|
+
|
264
|
+
1. Edit [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) and increment `VERSION`. Commit and push to rapid7/recog master.
|
265
|
+
|
266
|
+
1. Run `rake release`. Among other things, this creates the new gem, uploads it to Rubygems and tags the release with a tag like `v<VERSION>`, where `<VERSION>` is replaced with the version from `version.rb`. For example, if you release version 1.2.3 of the gem, the tag will be `v1.2.3`.
|
267
|
+
|
268
|
+
1. If your default remote repository is not `rapid7/recog`, you must ensure that the tags created in the previous step are also pushed to the right location(s). For example, if `origin` is your fork of recog and `upstream` is `rapid7/master`, you should run `git push --tags --dry-run upstream` to confirm what tags will be pushed and then `git push --tags upstream` to push the tags.
|
269
|
+
|
270
|
+
[^back to top](#contributing-to-recog)
|
data/Gemfile
CHANGED
@@ -1,13 +1,10 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
gemspec
|
3
|
+
gemspec name: 'recog'
|
4
4
|
|
5
5
|
gem 'nokogiri'
|
6
6
|
|
7
7
|
group :test do
|
8
8
|
gem 'rake'
|
9
|
-
gem '
|
10
|
-
gem 'cucumber', '~> 1.3.8'
|
11
|
-
gem 'aruba', '~> 0.5.3'
|
12
|
-
gem 'regexp_parser', '~> 0.2.0'
|
9
|
+
gem 'regexp_parser'
|
13
10
|
end
|
data/README.md
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# Recog: A Recognition Framework
|
2
|
+
|
2
3
|
[![Gem Version](https://badge.fury.io/rb/recog.svg)](http://badge.fury.io/rb/recog)
|
3
4
|
[![Build Status](https://travis-ci.org/rapid7/recog.svg?branch=master)](https://travis-ci.org/rapid7/recog)
|
4
5
|
|
@@ -7,6 +8,13 @@ Recog is a framework for identifying products, services, operating systems, and
|
|
7
8
|
|
8
9
|
Recog is open source, please see the [LICENSE](https://raw.githubusercontent.com/rapid7/recog/master/LICENSE) file for more information.
|
9
10
|
|
11
|
+
## Table of Contents
|
12
|
+
|
13
|
+
1. [Installation](#installation)
|
14
|
+
1. [Maturity](#maturity)
|
15
|
+
1. [Fingerprints](#fingerprints)
|
16
|
+
1. [Contributing](#contributing)
|
17
|
+
|
10
18
|
## Installation
|
11
19
|
|
12
20
|
Recog consists of both XML fingerprint files and an assortment of code, mostly in Ruby, that makes it easy to develop, test, and use the contained fingerprints. In order to use the included ruby code, a recent version of Ruby (2.31+) is required, along with Rubygems and the `bundler` gem. Once these dependencies are in place, use the following commands to grab the latest source code and install any additional dependencies.
|
@@ -17,10 +25,14 @@ $ cd recog
|
|
17
25
|
$ bundle install
|
18
26
|
```
|
19
27
|
|
28
|
+
[^back to top](#recog-a-recognition-framework)
|
29
|
+
|
20
30
|
## Maturity
|
21
31
|
|
22
32
|
Please note that while the XML fingerprints themselves are quite stable and well-tested, the Ruby codebase in Recog is still fairly new and subject to change quickly. Please contact us (research[at]rapid7.com) before leveraging the Recog code within any production projects.
|
23
33
|
|
34
|
+
[^back to top](#recog-a-recognition-framework)
|
35
|
+
|
24
36
|
## Fingerprints
|
25
37
|
|
26
38
|
The fingerprints within Recog are stored in XML files, each of which is designed to match a specific protocol response string or field. For example, the file [ssh_banners.xml](https://github.com/rapid7/recog/blob/master/xml/ssh_banners.xml) can determine the os, vendor, and sometimes hardware product by matching the initial SSH daemon banner string.
|
@@ -64,22 +76,12 @@ The `example` string can be base64 encoded to permit the use of unprintable char
|
|
64
76
|
</example>
|
65
77
|
````
|
66
78
|
|
67
|
-
|
68
|
-
|
69
|
-
Once a fingerprint has been added, the `example` entries can be tested by executing `bin/recog_verify` against the fingerprint file:
|
79
|
+
[^back to top](#recog-a-recognition-framework)
|
70
80
|
|
71
|
-
|
72
|
-
$ bin/recog_verify xml/ssh_banners.xml
|
73
|
-
```
|
74
|
-
|
75
|
-
Matches can be tested on the command-line in a similar fashion:
|
76
|
-
|
77
|
-
```shell
|
78
|
-
$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
|
79
|
-
MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
|
80
|
-
```
|
81
|
+
## Contributing
|
81
82
|
|
82
|
-
|
83
|
+
The users and maintainers of Recog would greatly appreciate any contributions
|
84
|
+
you can make to the project. For guidelines and instructions please see
|
85
|
+
[CONTRIBUTING.MD](CONTRIBUTING.md)
|
83
86
|
|
84
|
-
|
85
|
-
* Create regular expressions that allow for flexible version number matching. This ensures greater probability of matching a product. For example, all known public releases of a product report either `major.minor` or `major.minor.build` format version numbers. If the fingerprint strictly matches this version number format, it would fail to match a modified build of the product that reports only a `major` version number format.
|
87
|
+
[^back to top](#recog-a-recognition-framework)
|
data/bin/recog_cleanup
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
|
4
|
+
require 'optparse'
|
5
|
+
require 'ostruct'
|
6
|
+
require 'recog'
|
7
|
+
|
8
|
+
# Cleanup trailing whitespace around fingerprints
|
9
|
+
Dir[ File.expand_path(File.join(File.dirname(__FILE__), "..", "xml")) + "/*.xml" ].each do |f|
|
10
|
+
data = File.read(f).
|
11
|
+
gsub(/\s+$/, ''). # Trailing whitespace and empty lines
|
12
|
+
gsub("</fingerprint>", "</fingerprint>\n"). # Every fingerprint should have an empty line after it
|
13
|
+
gsub("-->", "-->\n") # Every comment should have an empty line after it
|
14
|
+
|
15
|
+
File.write(f, data)
|
16
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
|
4
|
+
require 'optparse'
|
5
|
+
require 'ostruct'
|
6
|
+
require 'recog'
|
7
|
+
|
8
|
+
def load_identifiers(path)
|
9
|
+
res = {}
|
10
|
+
File.readlines(path).map{|line| line.strip}.each do |ident|
|
11
|
+
res[ident] = true
|
12
|
+
end
|
13
|
+
return res
|
14
|
+
end
|
15
|
+
|
16
|
+
def write_identifiers(vals, path)
|
17
|
+
res = []
|
18
|
+
vals.each_pair do |k,v|
|
19
|
+
res = res.push(k)
|
20
|
+
end
|
21
|
+
res = res.map{|x| x.strip}.select{|x| x.length > 0}.sort.uniq
|
22
|
+
File.write(path, res.join("\n") + "\n")
|
23
|
+
end
|
24
|
+
|
25
|
+
bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
|
26
|
+
|
27
|
+
options = OpenStruct.new(write: false)
|
28
|
+
option_parser = OptionParser.new do |opts|
|
29
|
+
opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
|
30
|
+
opts.separator "Verifies that each fingerprint asserts known identifiers."
|
31
|
+
opts.separator ""
|
32
|
+
opts.separator "Options"
|
33
|
+
|
34
|
+
opts.on("-w", "--write") do
|
35
|
+
options.write = true
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("-h", "--help", "Show this message.") do
|
39
|
+
puts opts
|
40
|
+
exit
|
41
|
+
end
|
42
|
+
end
|
43
|
+
option_parser.parse!(ARGV)
|
44
|
+
|
45
|
+
if ARGV.empty?
|
46
|
+
$stderr.puts 'Missing XML fingerprint files'
|
47
|
+
puts option_parser
|
48
|
+
exit(1)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Load the unique identifiers
|
52
|
+
vendors = load_identifiers(File.join(bdir, "vendor.txt"))
|
53
|
+
os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
|
54
|
+
os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
|
55
|
+
os_family = load_identifiers(File.join(bdir, "os_family.txt"))
|
56
|
+
os_device = load_identifiers(File.join(bdir, "os_device.txt"))
|
57
|
+
hw_prod = load_identifiers(File.join(bdir, "hw_product.txt"))
|
58
|
+
hw_family = load_identifiers(File.join(bdir, "hw_family.txt"))
|
59
|
+
hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
|
60
|
+
svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
|
61
|
+
svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
|
62
|
+
|
63
|
+
|
64
|
+
ARGV.each do |arg|
|
65
|
+
Dir.glob(arg).each do |file|
|
66
|
+
ndb = Recog::DB.new(file)
|
67
|
+
ndb.fingerprints.each do |f|
|
68
|
+
f.params.each do |k,v|
|
69
|
+
paramIndex, val = v
|
70
|
+
next if paramIndex != 0
|
71
|
+
next if val.index("{") != -1
|
72
|
+
next if val.strip == ""
|
73
|
+
case k
|
74
|
+
when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
|
75
|
+
if ! vendors[val]
|
76
|
+
puts "VENDOR MISSING: #{val}"
|
77
|
+
vendors[val] = true
|
78
|
+
end
|
79
|
+
when "os.arch"
|
80
|
+
if ! os_arch[val]
|
81
|
+
puts "OS ARCH MISSING: #{val}"
|
82
|
+
os_arch[val] = true
|
83
|
+
end
|
84
|
+
when "os.product"
|
85
|
+
if ! os_prod[val]
|
86
|
+
puts "OS PRODUCT MISSING: #{val}"
|
87
|
+
os_prod[val] = true
|
88
|
+
end
|
89
|
+
when "os.family"
|
90
|
+
if ! os_family[val]
|
91
|
+
puts "OS FAMILY MISSING: #{val}"
|
92
|
+
os_family[val] = true
|
93
|
+
end
|
94
|
+
when "os.device"
|
95
|
+
if ! os_device[val]
|
96
|
+
puts "OS DEVICE MISSING: #{val}"
|
97
|
+
os_device[val] = true
|
98
|
+
end
|
99
|
+
when "hw.product"
|
100
|
+
if ! hw_prod[val]
|
101
|
+
puts "HW PRODUCT MISSING: #{val}"
|
102
|
+
hw_prod[val] = true
|
103
|
+
end
|
104
|
+
when "hw.family"
|
105
|
+
if ! hw_family[val]
|
106
|
+
puts "HW FAMILY MISSING: #{val}"
|
107
|
+
hw_family[val] = true
|
108
|
+
end
|
109
|
+
when "hw.device"
|
110
|
+
if ! hw_device[val]
|
111
|
+
puts "HW DEVICE MISSING: #{val}"
|
112
|
+
hw_device[val] = true
|
113
|
+
end
|
114
|
+
when "service.product"
|
115
|
+
if ! svc_prod[val]
|
116
|
+
puts "SERVICE PRODUCT MISSING: #{val}"
|
117
|
+
svc_prod[val] = true
|
118
|
+
end
|
119
|
+
when "service.family"
|
120
|
+
if ! svc_family[val]
|
121
|
+
puts "SERVICE FAMILY MISSING: #{val}"
|
122
|
+
svc_family[val] = true
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
exit if ! options.write
|
131
|
+
|
132
|
+
# Write back the unique identifiers
|
133
|
+
write_identifiers(vendors, File.join(bdir, "vendor.txt"))
|
134
|
+
write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
|
135
|
+
write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
|
136
|
+
write_identifiers(os_family, File.join(bdir, "os_family.txt"))
|
137
|
+
write_identifiers(os_device, File.join(bdir, "os_device.txt"))
|
138
|
+
write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
|
139
|
+
write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
|
140
|
+
write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
|
141
|
+
write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
|
142
|
+
write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
|