recog 2.3.22 → 3.0.2

data/.github/SECURITY.md

@@ -1,35 +0,0 @@
-# Reporting security issues
-
-Thanks for your interest in making Recog more secure! If you feel
-that you have found a security issue involving Metasploit, Meterpreter,
-Recog, or any other Rapid7 open source project, you are welcome to let
-us know in the way that's most comfortable for you.
-
-## Via ZenDesk
-
-You can click on the big blue button at [Rapid7's Vulnerability
-Disclosure][r7-vulns] page, which will get you to our general
-vulnerability reporting system. While this does require a (free) ZenDesk
-account to use, you'll get regular updates on your issue as our software
-support teams work through it. As it happens [that page][r7-vulns] also
-will tell you what to expect when it comes to reporting vulns, how fast
-we'll fix and respond, and all the rest, so it's a pretty good read
-regardless.
-
-## Via email
-
-If you're more of a traditionalist, you can email your finding to
-security@rapid7.com. If you like, you can use our [PGP key][pgp] to
-encrypt your messages, but we certainly don't mind cleartext reports
-over email.
-
-## NOT via GitHub Issues
-
-Please don't! Disclosing security vulnerabilities to public bug trackers
-is kind of mean, even when it's well-intentioned, since you end up
-dropping 0-day on pretty much everyone right out of the gate. We'd prefer
-you didn't!
-
-[r7-vulns]:https://www.rapid7.com/security/disclosure/
-[pgp]:https://keybase.io/rapid7/pgp_keys.asc?fingerprint=9a90aea0576cbcafa39c502ba5e16807959d3eda
-

data/.github/dependabot.yml

@@ -1,8 +0,0 @@
-version: 2
-updates:
-- package-ecosystem: bundler
-  directory: "/"
-  schedule:
-    interval: daily
-    time: "11:00"
-  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -1,26 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-jobs:
-  test:
-    name: 'Ruby: ${{ matrix.ruby-version }}'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Run tests
-      run: |
-        bundle exec rake --version
-        bundle exec rake tests
-      env:
-        JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G

data/.github/workflows/verify.yml

@@ -1,89 +0,0 @@
-name: Verify
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - 'xml/**.xml'
-  pull_request:
-    paths:
-      - 'xml/**.xml'
-
-jobs:
-  standardize:
-    name: 'Standardize'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-      - name: Run recog standardize
-        run: bundle exec bin/recog_standardize xml/*.xml
-  ruby-verify:
-    name: 'Ruby Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Ruby implementation
-        uses: actions/checkout@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-      - name: Run recog verify
-        run: bundle exec recog_verify --no-warnings xml/*.xml
-  java-verify:
-    name: 'Java Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Java implementation
-        uses: actions/checkout@v2
-        with:
-          repository: rapid7/recog-java
-      - name: Checkout recog content
-        uses: actions/checkout@v2
-        with:
-          path: recog-content
-      - uses: actions/setup-java@v2
-        with:
-          distribution: zulu
-          java-version: '17'
-      - name: Cache Maven packages
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-m2
-      - name: Build with Maven
-        run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
-      - name: Run recog verify
-        run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
-  go-verify:
-    name: 'Go Verify'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - name: Checkout Go implementation
-        uses: actions/checkout@v2
-        with:
-          repository: RumbleDiscovery/recog-go
-      - name: Checkout recog content
-        uses: actions/checkout@v2
-        with:
-          path: recog-content
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '^1.17.1'
-      - name: Run recog verify
-        run: go run cmd/recog_verify/main.go recog-content/xml/

data/.gitignore

@@ -1,23 +0,0 @@
-# Ruby and tooling specific
-.yardoc
-coverage/
-doc/
-pkg/
-
-/Gemfile.lock
-
-#Python specific
-venv
-
-# IDE specific
-.vscode/
-.idea
-
-# Misc
-**/.DS_Store
-
-# CPE XML
-official-cpe-dictionary*.xml
-
-# CPE Remap Errors
-errors.txt

data/.rspec

@@ -1,3 +0,0 @@
---color
---warnings
---require spec_helper

data/.ruby-gemset

@@ -1 +0,0 @@
-recog

data/.ruby-version

@@ -1 +0,0 @@
-2.6.6

data/.snyk

@@ -1,10 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-python: 3.6.0
-version: v1.14.1
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-PYTHON-PYYAML-590151:
-    - pyyaml:
-        reason: Project doesn't use vulnerable code path.
-        expires: 2021-06-01T00:00:00.000Z
-patch: {}

data/.travis.yml

@@ -1,25 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-rvm:
-  - '2.5.8'
-  - '2.6.6'
-  - 'jruby-9.1.9.0'
-jdk:
-  - openjdk8
-matrix:
-  allow_failures:
-    - rvm: 'jruby-9.1.9.0'
-before_install:
-  - "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
-  - rake --version
-before_script:
-  - bundle exec rake --version
-script: bundle exec rake tests
-env:
-  global:
-    - CI="travis"
-    - JRUBY_OPTS="--server -J-Xms512m -J-Xmx2G"
-branches:
-  only:
-    - master

data/CONTRIBUTING.md

@@ -1,276 +0,0 @@
-# Contributing to Recog
-
-The users and maintainers of Recog would greatly appreciate any contributions
-you can make to the project.  These contributions typically come in the form of
-filed bugs/issues or pull requests (PRs).  These contributions routinely result
-in new versions of the [recog gem](https://rubygems.org/gems/recog) being
-released.  The process for everything is described below.
-
-## Table of Contents
-
-1. [Contributing Issues / Bug Reports](#contributing-issues-/-bug-reports)
-1. [Contributing Code](#contributing-code)
-    1. [Fork and Clone](#fork-and-clone)
-    1. [Branch and Improve](#branch-and-improve)
-    1. [Testing](#testing)
-1. [Fingerprints](#fingerprints)
-    1. [Best Practices](#best-practices)
-    1. [Fingerprint Testing](#fingerprint-testing)
-    1. [Updating CPEs](#updating-cpes)
-1. [Project Operations](#project-operations)
-    1. [Landing PRs](#landing-prs)
-    1. [Releasing New Versions](#releasing-new-versions)
-
-## Contributing Issues / Bug Reports
-
-If you encounter any bugs or problems with Recog, please file them
-[here](https://github.com/rapid7/recog/issues/new), providing as much detail as
-possible.  If the bug is straight-forward enough and you understand the fix for
-the bug well enough, you may take the simpler, less-paperwork route and simply
-fill a PR with the fix and the necessary details.
-
-[^back to top](#contributing-to-recog)
-
-## Contributing Code
-
-Recog uses a model nearly identical to that of
-[Metasploit](https://github.com/rapid7/metasploit-framework) as outlined
-[here](https://github.com/rapid7/metasploit-framework/wiki/Setting-Up-a-Metasploit-Development-Environment),
-at least from a ```git``` perspective.  If you've been through that process
-(or, even better, you've been through it many times with many people), you can
-do exactly what you did for Metasploit but with Recog and ignore the rest of
-this document.
-
-On the other hand, if you haven't, read on!
-
-[^back to top](#contributing-to-recog)
-
-### Fork and Clone
-
-Generally, this should only need to be done once, or if you need to start over.
-
-1. Fork Recog: Visit https://github.com/rapid7/recog and click Fork,
-   selecting your github account if prompted
-1. Clone `git@github.com:<your-github-username>/recog.git`, replacing
-`<your-github-username>` with, you guessed it, your Github username.
-1. Add the master Recog repository as your upstream:
-
-     ```bash
-    git remote add upstream git://github.com/rapid7/recog.git
-    ```
-
-1. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream.  It should look something like the following, in particular the second `fetch` option:
-
-    ```bash
-     [remote "upstream"]
-      url = git@github.com:rapid7/recog.git
-      fetch = +refs/heads/*:refs/remotes/upstream/*
-      fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
-     ```
-
-1. Fetch the latest revisions, including PRs:
-
-    ```bash
-    git fetch --all
-    ```
-
-1. Set up git hooks to help identify potential issues with your contributions:
-
-    ```bash
-    ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
-    ```
-
-[^back to top](#contributing-to-recog)
-
-### Branch and Improve
-
-If you have a contribution to make, first create a branch to contain your
-work.  The name is yours to choose, however generally it should roughly
-describe what you are doing.  In this example, and from here on out, the
-branch will be FOO, but you should obviously change this:
-
-```bash
-git fetch --all
-git checkout master
-git rebase upstream/master
-git checkout -b FOO
-```
-
-Now, make your changes, commit as necessary with useful commit messages.
-
-Please note that changes to [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) in PRs are almost never necessary.
-
-Now push your changes to your fork:
-
-```bash
-git push origin FOO
-```
-
-Finally, submit the PR.  Navigate to ```https://github.com/<your-github-username>/recog/compare/FOO```, fill in the details and submit.
-
-[^back to top](#contributing-to-recog)
-
-### Testing
-
-When your PR is submitted, it will be automatically subjected to the full run of tests in [Travis](https://travis-ci.org/rapid7/recog/), however you are encourage to perform testing _before_ submitting the PR.  To do this, simply run `rake tests`.
-
-[^back to top](#contributing-to-recog)
-
-## Fingerprints
-
-### Best Practices
-
-* Create a single fingerprint for each product as long as the pattern remains clear and readable. If that is not possible, the pattern should be logically decomposed into additional fingerprints.
-
-* Create regular expressions that allow for flexible version number matching. This ensures greater probability of matching a product. For example, all known public releases of a product report either `major.minor` or `major.minor.build` format version numbers. If the fingerprint strictly matches this version number format, it would fail to match a modified build of the product that reports only a `major` version number format.
-
-[^back to top](#contributing-to-recog)
-
-### Fingerprint Testing
-
-Once a fingerprint has been added, the `example` entries can be tested by executing `bin/recog_verify` against the fingerprint file:
-
-```shell
-bin/recog_verify xml/ssh_banners.xml
-```
-
-Matches can be tested on the command-line in a similar fashion:
-
-```shell
-$ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
-MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
-```
-
-[^back to top](#contributing-to-recog)
-
-
-### Standardizing Vendors, Products, and Services
-
-Given the number of fingerprints in Recog, it can be common for specific products, vendors, or services to be identified with different spellings and casing.
-To limit the creep of slightly-different-names, the `bin/recog_standardize` script can be used to extract all identifiers and merge them into the known lists.
-
-To get started, run the `recog_standardize` tool:
-```shell
-ruby bin/recog_standardize
-```
-
-Review any new additions to the text files under `identifiers/`. If any of these names are close to an existing name, update the offending fingerprint to use
-the existing name instead. Once the fingerprints are fixed, removed the "extra" names from the identifiers files, and run the tool again.
-
-
-[^back to top](#contributing-to-recog)
-
-
-### Updating CPEs
-
-There exists some automation to update the CPEs that might be asserted with
-some recog fingerprints.  This should be run periodically to ensure that all
-fingerprints that could have CPEs do, etc.
-
-First, setup a python3 venv:
-
-  ```bash
-  python3 -m venv venv
-  source venv/{bin,Scripts}/activate
-  pip install -r requirements.txt
-  ```
-
-Download the latest CPE 2.3 dictionary:
-
-```bash
-curl -o official-cpe-dictionary_v2.3.xml.gz https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz && \
-gunzip official-cpe-dictionary_v2.3.xml.gz
-```
-
-Run the CPE automation against every XML file:
-
-```bash
-# Update the CPEs (sequentially)
-ls xml/*.xml | xargs -i python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml 2>>errors.txt
-```
-
-You may want to use GNU `parallel` to speed things up:
-```bash
-# Update the CPEs (with GNU Parallel)
-ls xml/*.xml | parallel --gnu "python update_cpes.py {} official-cpe-dictionary_v2.3.xml cpe-remap.yaml"  2>>errors.txt
-```
-
-Clean up the whitespace across all fingerprints:
-```bash
-ruby bin/recog_cleanup
-```
-
-Any mismatched fingerprints will be listed in `errors.txt` for eventual
-maintenance.  The `cpe-remap.yaml` file can be used to map between
-vendor/product/etc differences between Recog and CPE, or to work around bugs in
-either.
-
-[^back to top](#contributing-to-recog)
-
-## Project Operations
-
-### Landing PRs
-
-(Note: this portion is a work-in-progress.  Please update it as things change)
-
-Much like with the process of submitting PRs, Recog's process for landing PRs
-is very similar to [Metasploit's process for landing
-PRs](https://github.com/rapid7/metasploit-framework/wiki/Landing-Pull-Requests).
-In short:
-
-1. Follow the "Fork and Clone" steps from above
-2. Update your `.git/config` to ensure that the `remote ["upstream"]` section is configured to pull both branches and PRs from upstream.  It should look something like the following, in particular the second `fetch` option:
-
-    ```bash
-     [remote "upstream"]
-      url = git@github.com:rapid7/recog.git
-      fetch = +refs/heads/*:refs/remotes/upstream/*
-      fetch = +refs/pull/*/head:refs/remotes/upstream/pr/*
-     ```
-
-3. Fetch the latest revisions, including PRs:
-
-    ```bash
-    git fetch --all
-    ```
-
-4. Checkout and branch the PR for testing.  Replace ```PR``` below with the actual PR # in question:
-
-    ```bash
-    git checkout -b landing-PR upstream/pr/PR
-    ```
-
-5. Test the PR (see the Testing section above)
-6. Merge with master, re-test, validate and push:
-
-    ```bash
-    git checkout -b upstream-master --track upstream/master
-    git merge -S --no-ff --edit landing-PR # merge the PR into upstream-master
-
-    # re-test if/as necessary
-    git push upstream upstream-master:master --dry-run # confirm you are pushing what you expect
-
-    git push upstream upstream-master:master # push upstream-master to upstream:master
-    ```
-
-7. If applicable, release a new version (see next section)
-
-[^back to top](#contributing-to-recog)
-
-### Releasing New Versions
-
-When Recog's critical parts are modified, for example its fingerprints or underlying supporting code, a new version _must_ eventually be released.  These new releases can then be optionally included in projects such as Metasploit or products such as Rapid7's Nexpose in a controlled manner.  Releases for non-functional updates such as updates to documentation are not necessary.
-
-When a new version of Recog is to be released, you _must_ follow the instructions below.
-
-1. If are not already a Recog project contributor for the Recog gem (you'd be listed [here under OWNERS](https://rubygems.org/gems/recog)), become one:
-   1. Get an account on [Rubygems](https://rubygems.org)
-   1. Contact one of the Recog project contributors (listed [here under OWNERS](https://rubygems.org/gems/recog) and have them add you to the Recog gem.  They'll need to run: `gem owner recog -a EMAIL`
-
-1. Edit [lib/recog/version.rb](https://github.com/rapid7/recog/blob/master/lib/recog/version.rb) and increment `VERSION`.  Commit and push to rapid7/recog master.
-
-1. Run `rake release`.  Among other things, this creates the new gem, uploads it to Rubygems and tags the release with a tag like `v<VERSION>`, where `<VERSION>` is replaced with the version from `version.rb`.  For example, if you release version 1.2.3 of the gem, the tag will be `v1.2.3`.
-
-1. If your default remote repository is not `rapid7/recog`, you must ensure that the tags created in the previous step are also pushed to the right location(s).  For example, if `origin` is your fork of recog and `upstream` is `rapid7/master`, you should run `git push --tags --dry-run upstream` to confirm what tags will be pushed and then `git push --tags upstream` to push the tags.
-
-[^back to top](#contributing-to-recog)

data/bin/recog_cleanup

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-# Cleanup trailing whitespace around fingerprints
-Dir[ File.expand_path(File.join(File.dirname(__FILE__), "..", "xml")) + "/*.xml" ].each do |f|
-    data = File.read(f).
-        gsub(/\s+$/, '').                           # Trailing whitespace and empty lines
-        gsub("</fingerprint>", "</fingerprint>\n"). # Every fingerprint should have an empty line after it
-        gsub("-->", "-->\n")                        # Every comment should have an empty line after it
-
-    File.write(f, data)
-end

data/bin/recog_export

@@ -1,81 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-def squash_lines(str)
-  str.split(/\n/).join(' ').gsub(/\s+/, ' ')
-end
-
-def export_text(options)
-end
-
-def export_ruby(options)
-  $stdout.puts "# Recog fingerprint database export [ #{File.basename(options.xml_file)} ] on #{Time.now.to_s}"
-  $stdout.puts "fp_str   = '' # Set this value to the match string"
-  $stdout.puts "fp_match = {} # Match results are stored here"
-  $stdout.puts ""
-  $stdout.puts "case fp_str"
-  options.db.fingerprints.each do |fp|
-    puts "  # #{squash_lines fp.name}"
-    puts "  when /#{fp.regex.to_s}/"
-    fp.tests.each do |test|
-      puts "    # Example: #{squash_lines test}"
-    end
-    fp.params.each_pair do |k,v|
-      if v[0] == 0
-        puts "    fp_match[#{k.inspect}] = #{v[1].inspect}"
-      else
-        puts "    fp_match[#{k.inspect}] = $#{v[0].to_s}"
-      end
-    end
-    puts ""
-  end
-  $stdout.puts "end"
-end
-
-
-options = OpenStruct.new(etype: :ruby)
-
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINTS_FILE"
-  opts.separator "Exports an XML fingerprint database to another format."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-t", "--type type", 
-          "Choose a type of export.",
-          "  [r]uby (default - export a ruby case statement with regular expressions)",
-          "  [t]ext (export a text description of the fingerprints)") do |etype|
-    case etype.downcase
-    when /^r/
-      options.etype = :ruby
-    when /^t/
-      options.etype = :text
-    end
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.count != 1
-  puts option_parser
-  exit
-end
-
-options.xml_file = ARGV.shift
-options.db = Recog::DB.new(options.xml_file)
-
-case options.etype
-when :ruby
-  export_ruby(options)
-when :text
-  export_text(options)
-end
-