recog 2.3.23 → 3.0.1

data/.github/workflows/ci.yml

@@ -1,26 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-jobs:
-  test:
-    name: 'Ruby: ${{ matrix.ruby-version }}'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.2.20.1', 'jruby']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Run tests
-      run: |
-        bundle exec rake --version
-        bundle exec rake tests
-      env:
-        JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G

data/.github/workflows/verify.yml

@@ -1,89 +0,0 @@
-name: Verify
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - 'xml/**.xml'
-  pull_request:
-    paths:
-      - 'xml/**.xml'
-
-jobs:
-  standardize:
-    name: 'Standardize'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-      - name: Run recog standardize
-        run: bundle exec bin/recog_standardize xml/*.xml
-  ruby-verify:
-    name: 'Ruby Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Ruby implementation
-        uses: actions/checkout@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-      - name: Run recog verify
-        run: bundle exec recog_verify --schema-location xml/fingerprints.xsd --no-warnings xml/*.xml
-  java-verify:
-    name: 'Java Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Java implementation
-        uses: actions/checkout@v2
-        with:
-          repository: rapid7/recog-java
-      - name: Checkout recog content
-        uses: actions/checkout@v2
-        with:
-          path: recog-content
-      - uses: actions/setup-java@v2
-        with:
-          distribution: zulu
-          java-version: '17'
-      - name: Cache Maven packages
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-m2
-      - name: Build with Maven
-        run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
-      - name: Run recog verify
-        run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
-  go-verify:
-    name: 'Go Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Go implementation
-        uses: actions/checkout@v2
-        with:
-          repository: RumbleDiscovery/recog-go
-      - name: Checkout recog content
-        uses: actions/checkout@v2
-        with:
-          path: recog-content
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17.1'
-      - name: Run recog verify
-        run: go run cmd/recog_verify/main.go recog-content/xml/

data/.gitignore

@@ -1,23 +0,0 @@
-# Ruby and tooling specific
-.yardoc
-coverage/
-doc/
-pkg/
-
-/Gemfile.lock
-
-#Python specific
-venv
-
-# IDE specific
-.vscode/
-.idea
-
-# Misc
-**/.DS_Store
-
-# CPE XML
-official-cpe-dictionary*.xml
-
-# CPE Remap Errors
-errors.txt

data/.rspec

@@ -1,3 +0,0 @@
---color
---warnings
---require spec_helper

data/.ruby-gemset

@@ -1 +0,0 @@
-recog

data/.ruby-version

@@ -1 +0,0 @@
-2.6.6

data/.snyk

@@ -1,10 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-python: 3.6.0
-version: v1.14.1
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-PYTHON-PYYAML-590151:
-    - pyyaml:
-        reason: Project doesn't use vulnerable code path.
-        expires: 2021-06-01T00:00:00.000Z
-patch: {}

data/.travis.yml

@@ -1,25 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-rvm:
-  - '2.5.8'
-  - '2.6.6'
-  - 'jruby-9.1.9.0'
-jdk:
-  - openjdk8
-matrix:
-  allow_failures:
-    - rvm: 'jruby-9.1.9.0'
-before_install:
-  - "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
-  - rake --version
-before_script:
-  - bundle exec rake --version
-script: bundle exec rake tests
-env:
-  global:
-    - CI="travis"
-    - JRUBY_OPTS="--server -J-Xms512m -J-Xmx2G"
-branches:
-  only:
-    - master

data/.vscode/bin/monitor-recog-fingerprints.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-ARGS=()
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        -s|--schema-location)
-            VALIDATE_SCHEMA="--schema-location $2"
-            shift
-            shift
-            ;;
-        -*|--*)
-            echo "Unknown option $1"
-            exit 1
-            ;;
-        *)
-            ARGS+=("$1")
-            shift
-            ;;
-    esac
-done
-set -- "${ARGS[@]}"
-
-if [ $# -eq 0 ]
-then
-    echo "Usage: $(basename $0) [--schema-location SCHEMA_LOCATION] <xml fingerprint directory>"
-    exit 1
-fi
-
-if [ ! -d "$1" ]
-then
-    echo "The XML fingerprint file directory must be supplied."
-    exit 1
-fi
-
-bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
-
-if ! type fswatch &>/dev/null;
-then 
-    echo "'fswatch' is required to monitor fingerprint files for changes and update the editor."
-    echo "See: https://emcrisostomo.github.io/fswatch/ or install with:"
-    echo " MacOS Homebrew: brew install fswatch"
-    echo " Ubuntu/Debian:  apt install fswatch"
-    echo
-    echo "Otherwise, you can re-run this task using the Visual Studio Code command palette"
-    exit 1
-fi
-
-echo "Waiting for changes..."
-fswatch -0 $1 | while read -d "" event; do {
-    echo "Changes detected, validating: ${event}"
-    # TODO: VSCode doesn't support individual/incremental updates to files yet.
-    bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
-    echo "Waiting for changes..."
-}; done

data/.vscode/extensions.json

@@ -1,5 +0,0 @@
-{
-    "recommendations": [
-        "redhat.vscode-xml"
-    ]
-}

data/.vscode/settings.json

@@ -1,8 +0,0 @@
-{
-  "xml.fileAssociations": [
-    {
-        "pattern": "xml/*.xml",
-        "systemId": "xml/fingerprints.xsd"
-    }
- ]
-}

data/.vscode/tasks.json

@@ -1,77 +0,0 @@
-{
-    "version": "2.0.0",
-    "tasks": [
-        {
-            "label": "Recog Verify - Background Monitor",
-            "command": ".vscode/bin/monitor-recog-fingerprints.sh",
-            "args": [
-                "--schema-location",
-                "xml/fingerprints.xsd",
-                "${workspaceFolder}/xml"
-            ],
-            "windows": {
-                "command": ""
-            },
-            "type": "process",
-            "isBackground": true,
-            "problemMatcher": {
-                "owner": "recog",
-                "fileLocation": [
-                    "absolute"
-                ],
-                "pattern": {
-                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
-                    "file": 1,
-                    "severity": 3,
-                    "message": 4,
-                    "location": 2
-                },
-                "background": {
-                    "activeOnStart": true,
-                    "beginsPattern": "^Changes detected",
-                    "endsPattern": "^Waiting for changes"
-                },
-            },
-            "presentation": {
-                "reveal": "always",
-                "revealProblems": "onProblem"
-            },
-            "runOptions": {
-                "runOn": "folderOpen"
-            }
-        },
-        {
-            "label": "Recog Verify",
-            "command": "bin/recog_verify",
-            "args": [
-                "--schema-location",
-                "xml/fingerprints.xsd",
-                "${workspaceFolder}/xml/*.xml"
-            ],
-            "windows": {
-                "command": ""
-            },
-            "type": "process",
-            "problemMatcher": {
-                "owner": "recog",
-                "fileLocation": [
-                    "absolute"
-                ],
-                "pattern": {
-                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
-                    "file": 1,
-                    "severity": 3,
-                    "message": 4,
-                    "location": 2
-                }
-            },
-            "presentation": {
-                "reveal": "always",
-                "revealProblems": "onProblem"
-            },
-            "runOptions": {
-                "runOn": "folderOpen"
-            }
-        }
-    ]
-}

data/CONTRIBUTING.md

@@ -1,278 +0,0 @@
-# Contributing to Recog
-
-The users and maintainers of Recog would greatly appreciate any contributions
-you can make to the project.  These contributions typically come in the form of
-filed bugs/issues or pull requests (PRs).  These contributions routinely result
-in new versions of the [recog gem](https://rubygems.org/gems/recog) being
-released.  The process for everything is described below.
-
-## Table of Contents
-
-1. [Contributing Issues / Bug Reports](#contributing-issues-/-bug-reports)
-1. [Contributing Code](#contributing-code)
-    1. [Fork and Clone](#fork-and-clone)
-    1. [Branch and Improve](#branch-and-improve)
-    1. [Testing](#testing)
-1. [Fingerprints](#fingerprints)
-    1. [Best Practices](#best-practices)
-    1. [Fingerprint Testing](#fingerprint-testing)
-    1. [Updating CPEs](#updating-cpes)
-1. [Project Operations](#project-operations)
-    1. [Landing PRs](#landing-prs)
-    1. [Releasing New Versions](#releasing-new-versions)
-
-## Contributing Issues / Bug Reports
-
-If you encounter any bugs or problems with Recog, please file them
-[here](https://github.com/rapid7/recog/issues/new), providing as much detail as
-possible.  If the bug is straight-forward enough and you understand the fix for
-the bug well enough, you may take the simpler, less-paperwork route and simply
-fill a PR with the fix and the necessary details.
-
-[^back to top](#contributing-to-recog)
-
-## Contributing Code
-
-Recog uses a model nearly identical to that of
-[Metasploit](https://github.com/rapid7/metasploit-framework) as outlined
-[here](https://github.com/rapid7/metasploit-framework/wiki/Setting-Up-a-Metasploit-Development-Environment),
-at least from a ```git``` perspective.  If you've been through that process
-(or, even better, you've been through it many times with many people), you can
-do exactly what you did for Metasploit but with Recog and ignore the rest of
-this document.
-
-On the other hand, if you haven't, read on!
-
-[^back to top](#contributing-to-recog)
-
-### Fork and Clone
-
-Generally, this should only need to be done once, or if you need to start over.
-
-1. Fork Recog: Visit https://github.com/rapid7/recog and click Fork,
-   selecting your github account if prompted
-1. Clone `git@github.com:<your-github-username>/recog.git`, replacing
-`<your-github-username>` with, you guessed it, your Github username.
-1. Add the master Recog repository as your upstream:
-
-     ```bash
-    git remote add upstream git://github.com/rapid7/recog.git
-    ```
-
-1. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream.  It should look something like the following, in particular the second `fetch` option:
-
-    ```bash
-     [remote "upstream"]
-      url = git@github.com:rapid7/recog.git
-      fetch = +refs/heads/*:refs/remotes/upstream/*
-      fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
-     ```
-
-1. Fetch the latest revisions, including PRs:
-
-    ```bash
-    git fetch --all
-    ```
-
-1. Set up git hooks to help identify potential issues with your contributions:
-
-    ```bash
-    ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
-    ```
-
-[^back to top](#contributing-to-recog)
-
-### Branch and Improve
-
-If you have a contribution to make, first create a branch to contain your
-work.  The name is yours to choose, however generally it should roughly
-describe what you are doing.  In this example, and from here on out, the
-branch will be FOO, but you should obviously change this:
-
-```bash
-git fetch --all
-git checkout master
-git rebase upstream/master
-git checkout -b FOO
-```
-
-Now, make your changes, commit as necessary with useful commit messages.
-
-Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) in PRs are almost never necessary.
-
-Now push your changes to your fork:
-
-```bash
-git push origin FOO
-```
-
-Finally, submit the PR.  Navigate to ```https://github.com/<your-github-username>/recog/compare/FOO```, fill in the details and submit.
-
-[^back to top](#contributing-to-recog)
-
-### Testing
-
-When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR.  To do this, simply run `rake tests`.
-
-[^back to top](#contributing-to-recog)
-
-## Fingerprints
-
-### Best Practices
-
-* Create a single fingerprint for each product as long as the pattern remains clear and readable. If that is not possible, the pattern should be logically decomposed into additional fingerprints.
-
-* Create regular expressions that allow for flexible version number matching. This ensures greater probability of matching a product. For example, all known public releases of a product report either `major.minor` or `major.minor.build` format version numbers. If the fingerprint strictly matches this version number format, it would fail to match a modified build of the product that reports only a `major` version number format.
-
-[^back to top](#contributing-to-recog)
-
-### Fingerprint Testing
-
-Once a fingerprint has been added, the `example` entries can be tested by executing `bin/recog_verify` against the fingerprint file:
-
-```shell
-bin/recog_verify xml/ssh_banners.xml
-```
-
-Matches can be tested on the command-line in a similar fashion:
-
-```shell
-$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
-MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
-```
-
-Additionally, in Visual Studio Code, there is a task (.vscode/tasks.json) which will automatically run recog_verify in the background to watch all the XML fingerprint files (under the xml/ subdirectory of this repository). Additionally, if [fswatch](https://github.com/emcrisostomo/fswatch) is installed, whenever XML fingerprint files are added or modified this task will automatically update the Visual Studio Code user interface and highlight any errors or warnings discovered through recog_verify on the correct file/line. You can also manually run the task by bringing up the Visual Studio Code command menu (cmd + shift + P on mac, or ctrl + shift + P for linux/windows) -> Tasks: Run Task -> Recog Verify). Note that in order for the task to run successfully, you must have a valid ruby installed on your PATH with the gems from `bundle install` installed using bundler for that ruby engine. JRuby is not supported as it has issues related to line numbering due to a bug in Nokogiri.
-
-[^back to top](#contributing-to-recog)
-
-
-### Standardizing Vendors, Products, and Services
-
-Given the number of fingerprints in Recog, it can be common for specific products, vendors, or services to be identified with different spellings and casing.
-To limit the creep of slightly-different-names, the `bin/recog_standardize` script can be used to extract all identifiers and merge them into the known lists.
-
-To get started, run the `recog_standardize` tool:
-```shell
-ruby bin/recog_standardize
-```
-
-Review any new additions to the text files under `identifiers/`. If any of these names are close to an existing name, update the offending fingerprint to use
-the existing name instead. Once the fingerprints are fixed, removed the "extra" names from the identifiers files, and run the tool again.
-
-
-[^back to top](#contributing-to-recog)
-
-
-### Updating CPEs
-
-There exists some automation to update the CPEs that might be asserted with
-some recog fingerprints.  This should be run periodically to ensure that all
-fingerprints that could have CPEs do, etc.
-
-First, setup a python3 venv:
-
-  ```bash
-  python3 -m venv venv
-  source venv/{bin,Scripts}/activate
-  pip install -r requirements.txt
-  ```
-
-Download the latest CPE 2.3 dictionary:
-
-```bash
-curl -o official-cpe-dictionary_v2.3.xml.gz https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz && \
-gunzip official-cpe-dictionary_v2.3.xml.gz
-```
-
-Run the CPE automation against every XML file:
-
-```bash
-# Update the CPEs (sequentially)
-ls xml/*.xml | xargs -i python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml 2>>errors.txt
-```
-
-You may want to use GNU `parallel` to speed things up:
-```bash
-# Update the CPEs (with GNU Parallel)
-ls xml/*.xml | parallel --gnu "python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml"  2>>errors.txt
-```
-
-Clean up the whitespace across all fingerprints:
-```bash
-ruby bin/recog_cleanup
-```
-
-Any mismatched fingerprints will be listed in `errors.txt` for eventual
-maintenance.  The `cpe-remap.yaml` file can be used to map between
-vendor/product/etc differences between Recog and CPE, or to work around bugs in
-either.
-
-[^back to top](#contributing-to-recog)
-
-## Project Operations
-
-### Landing PRs
-
-(Note: this portion is a work-in-progress.  Please update it as things change)
-
-Much like with the process of submitting PRs, Recog's process for landing PRs
-is very similar to [Metasploit's process for landing
-PRs](https://github.com/rapid7/metasploit-framework/wiki/Landing-Pull-Requests).
-In short:
-
-1. Follow the "Fork and Clone" steps from above
-2. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream.  It should look something like the following, in particular the second `fetch` option:
-
-    ```bash
-     [remote "upstream"]
-      url = git@github.com:rapid7/recog.git
-      fetch = +refs/heads/*:refs/remotes/upstream/*
-      fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
-     ```
-
-3. Fetch the latest revisions, including PRs:
-
-    ```bash
-    git fetch --all
-    ```
-
-4. Checkout and branch the PR for testing.  Replace ```PR``` below with the actual PR # in question:
-
-    ```bash
-    git checkout -b landing-PR upstream/pr/PR
-    ```
-
-5. Test the PR (see the Testing section above)
-6. Merge with master, re-test, validate and push:
-
-    ```bash
-    git checkout -b upstream-master --track upstream/master
-    git merge -S --no-ff --edit landing-PR # merge the PR into upstream-master
-
-    # re-test if/as necessary
-    git push upstream upstream-master:master --dry-run # confirm you are pushing what you expect
-
-    git push upstream upstream-master:master # push upstream-master to upstream:master
-    ```
-
-7. If applicable, release a new version (see next section)
-
-[^back to top](#contributing-to-recog)
-
-### Releasing New Versions
-
-When Recog's critical parts are modified, for example its fingerprints or underlying supporting code, a new version _must_ eventually be released.  These new releases can then be optionally included in projects such as Metasploit or products such as Rapid7's Nexpose in a controlled manner.  Releases for non-functional updates such as updates to documentation are not necessary.
-
-When a new version of Recog is to be released, you _must_ follow the instructions below.
-
-1. If are not already a Recog project contributor for the Recog gem (you'd be listed [here under OWNERS](https://rubygems.org/gems/recog)), become one:
-   1. Get an account on [Rubygems](https://rubygems.org)
-   1. Contact one of the Recog project contributors (listed [here under OWNERS](https://rubygems.org/gems/recog) and have them add you to the Recog gem.  They'll need to run: `gem owner recog -a EMAIL`
-
-1. Edit [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) and increment `VERSION`.  Commit and push to rapid7/recog master.
-
-1. Run `rake release`.  Among other things, this creates the new gem, uploads it to Rubygems and tags the release with a tag like `v<VERSION>`, where `<VERSION>` is replaced with the version from `version.rb`.  For example, if you release version 1.2.3 of the gem, the tag will be `v1.2.3`.
-
-1. If your default remote repository is not `rapid7/recog`, you must ensure that the tags created in the previous step are also pushed to the right location(s).  For example, if `origin` is your fork of recog and `upstream` is `rapid7/master`, you should run `git push --tags --dry-run upstream` to confirm what tags will be pushed and then `git push --tags upstream` to push the tags.
-
-[^back to top](#contributing-to-recog)

data/bin/recog_cleanup

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-# Cleanup trailing whitespace around fingerprints
-Dir[ File.expand_path(File.join(File.dirname(__FILE__), "..", "xml")) + "/*.xml" ].each do |f|
-    data = File.read(f).
-        gsub(/\s+$/, '').                           # Trailing whitespace and empty lines
-        gsub("</fingerprint>", "</fingerprint>\n"). # Every fingerprint should have an empty line after it
-        gsub("-->", "-->\n")                        # Every comment should have an empty line after it
-
-    File.write(f, data)
-end