recog 2.3.21 → 3.0.1

data/spec/lib/recog/fingerprint_spec.rb

@@ -75,7 +75,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[6]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -83,7 +83,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[7]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 
@@ -92,7 +92,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[8]) }
 
       it "identifies when a parameter defined by a capture group is not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String])
       end
     end
 
@@ -100,7 +100,7 @@ describe Recog::Fingerprint do
       let(:entry) { described_class.new(doc.xpath("//fingerprints/fingerprint")[9]) }
 
       it "identifies when two parameters defined by a capture groups are not included in one example" do
-        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:warn, String], [:warn, String])
+        expect { |unused| entry.verify_tests_have_capture_groups(&unused) }.to yield_successive_args([:fail, String], [:fail, String])
       end
     end
 

data/spec/lib/recog/verify_reporter_spec.rb

@@ -7,6 +7,7 @@ describe Recog::VerifyReporter do
   let(:summary_line) do
     "SUMMARY: Test completed with 1 successful, 1 warnings, and 1 failures"
   end
+  let(:path) { "fingerprint.xml" }
 
   subject { Recog::VerifyReporter.new(double(detail: false, quiet: false, warnings: true), formatter) }
 
@@ -21,12 +22,12 @@ describe Recog::VerifyReporter do
 
   describe "#report" do
     it "prints warnings" do
-      expect(formatter).to receive(:warning_message).with('a warning')
+      expect(formatter).to receive(:warning_message).with('WARN: a warning')
       run_report
     end
 
     it "prints failures" do
-      expect(formatter).to receive(:failure_message).with('a failure')
+      expect(formatter).to receive(:failure_message).with('FAIL: a failure')
       run_report
     end
 
@@ -49,12 +50,80 @@ describe Recog::VerifyReporter do
       end
 
       it "prints warnings" do
-        expect(formatter).to receive(:warning_message).with('   a warning')
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
         run_report
       end
 
       it "prints failures" do
-        expect(formatter).to receive(:failure_message).with('   a failure')
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
+        run_report
+      end
+
+      it "prints the fingerprint count" do
+        expect(formatter).to receive(:status_message).with("\nVerified 1 fingerprints:")
+        run_report
+      end
+
+      it "prints summary" do
+        expect(formatter).to receive(:failure_message).with(summary_line)
+        run_report
+      end
+
+      context "with no fingerprint tests" do
+        let(:tests) { [] }
+
+        it "does not print the name" do
+          expect(formatter).not_to receive(:status_message).with("\na name")
+          run_report
+        end
+      end
+    end
+
+    context "with fingerprint path" do
+
+      subject { Recog::VerifyReporter.new(double(detail: false, quiet: false, warnings: true), formatter, path) }
+
+      it "prints warnings" do
+        expect(formatter).to receive(:warning_message).with("#{path}: WARN: a warning")
+        run_report
+      end
+
+      it "prints failures" do
+        expect(formatter).to receive(:failure_message).with("#{path}: FAIL: a failure")
+        run_report
+      end
+
+      it "prints summary" do
+        expect(formatter).to receive(:failure_message).with("#{path}: #{summary_line}")
+        run_report
+      end
+    end
+
+    context "with fingerprint path and detail" do
+      subject { Recog::VerifyReporter.new(double(detail: true, quiet: false, warnings: true), formatter, path) }
+
+      it "prints the fingerprint path" do
+        expect(formatter).to receive(:status_message).with("\n#{path}:\n")
+        run_report
+      end
+
+      it "prints the fingerprint name" do
+        expect(formatter).to receive(:status_message).with("\na name")
+        run_report
+      end
+
+      it "prints successes" do
+        expect(formatter).to receive(:success_message).with('   passed')
+        run_report
+      end
+
+      it "prints warnings" do
+        expect(formatter).to receive(:warning_message).with('   WARN: a warning')
+        run_report
+      end
+
+      it "prints failures" do
+        expect(formatter).to receive(:failure_message).with('   FAIL: a failure')
         run_report
       end
 

data/spec/spec_helper.rb

@@ -1,6 +1,10 @@
+FINGERPRINT_DIR = File.expand_path(File.join('..', 'recog', 'xml'), __dir__)
+
+# setup code coverage
 require 'simplecov'
 SimpleCov.start
 
+require 'rspec'
 # This file was generated by the `rspec --init` command. Conventionally, all
 # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
 # The generated `.rspec` file contains `--require spec_helper` which will cause this

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: recog
 version: !ruby/object:Gem::Version
-  version: 2.3.21
+  version: 3.0.1
 platform: ruby
 authors:
 - Rapid7 Research
 autorequire:
-bindir: bin
+bindir: recog/bin
 cert_chain: []
-date: 2021-08-02 00:00:00.000000000 Z
+date: 2022-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: cucumber
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: aruba
   requirement: !ruby/object:Gem::Requirement
@@ -115,90 +101,94 @@ description: Recog is a framework for identifying products, services, operating
 email:
 - research@rapid7.com
 executables:
-- recog_cleanup
-- recog_export
 - recog_match
-- recog_standardize
-- recog_verify
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/ISSUE_TEMPLATE/bug_report.md"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
-- ".github/ISSUE_TEMPLATE/fingerprint_request.md"
-- ".github/PULL_REQUEST_TEMPLATE"
-- ".github/SECURITY.md"
-- ".github/workflows/ci.yml"
-- ".gitignore"
-- ".rspec"
-- ".ruby-gemset"
-- ".ruby-version"
-- ".snyk"
-- ".travis.yml"
 - ".yardopts"
-- CONTRIBUTING.md
 - COPYING
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
-- bin/recog_cleanup
-- bin/recog_export
-- bin/recog_match
-- bin/recog_standardize
-- bin/recog_verify
-- cpe-remap.yaml
-- features/data/failing_banners_fingerprints.xml
-- features/data/matching_banners_fingerprints.xml
-- features/data/multiple_banners_fingerprints.xml
-- features/data/no_tests.xml
-- features/data/sample_banner.txt
-- features/data/successful_tests.xml
-- features/data/tests_with_failures.xml
-- features/data/tests_with_warnings.xml
-- features/match.feature
-- features/support/aruba.rb
-- features/support/env.rb
-- features/verify.feature
-- identifiers/README.md
-- identifiers/fields.txt
-- identifiers/hw_device.txt
-- identifiers/hw_family.txt
-- identifiers/hw_product.txt
-- identifiers/os_architecture.txt
-- identifiers/os_device.txt
-- identifiers/os_family.txt
-- identifiers/os_product.txt
-- identifiers/service_family.txt
-- identifiers/service_product.txt
-- identifiers/vendor.txt
 - lib/recog.rb
 - lib/recog/db.rb
 - lib/recog/db_manager.rb
 - lib/recog/fingerprint.rb
 - lib/recog/fingerprint/regexp_factory.rb
 - lib/recog/fingerprint/test.rb
+- lib/recog/fingerprint_parse_error.rb
 - lib/recog/formatter.rb
 - lib/recog/match_reporter.rb
 - lib/recog/matcher.rb
 - lib/recog/matcher_factory.rb
 - lib/recog/nizer.rb
 - lib/recog/verifier.rb
-- lib/recog/verifier_factory.rb
 - lib/recog/verify_reporter.rb
 - lib/recog/version.rb
-- misc/convert_mysql_err
-- misc/order.xsl
 - recog.gemspec
-- requirements.txt
+- recog/bin/recog_match
+- recog/xml/apache_modules.xml
+- recog/xml/apache_os.xml
+- recog/xml/architecture.xml
+- recog/xml/dhcp_vendor_class.xml
+- recog/xml/dns_versionbind.xml
+- recog/xml/favicons.xml
+- recog/xml/fingerprints.xsd
+- recog/xml/ftp_banners.xml
+- recog/xml/h323_callresp.xml
+- recog/xml/hp_pjl_id.xml
+- recog/xml/html_title.xml
+- recog/xml/http_cookies.xml
+- recog/xml/http_servers.xml
+- recog/xml/http_wwwauth.xml
+- recog/xml/imap_banners.xml
+- recog/xml/ldap_searchresult.xml
+- recog/xml/mdns_device-info_txt.xml
+- recog/xml/mdns_workstation_txt.xml
+- recog/xml/mysql_banners.xml
+- recog/xml/mysql_error.xml
+- recog/xml/nntp_banners.xml
+- recog/xml/ntp_banners.xml
+- recog/xml/operating_system.xml
+- recog/xml/pop_banners.xml
+- recog/xml/rsh_resp.xml
+- recog/xml/rtsp_servers.xml
+- recog/xml/sip_banners.xml
+- recog/xml/sip_user_agents.xml
+- recog/xml/smb_native_lm.xml
+- recog/xml/smb_native_os.xml
+- recog/xml/smtp_banners.xml
+- recog/xml/smtp_debug.xml
+- recog/xml/smtp_ehlo.xml
+- recog/xml/smtp_expn.xml
+- recog/xml/smtp_help.xml
+- recog/xml/smtp_mailfrom.xml
+- recog/xml/smtp_noop.xml
+- recog/xml/smtp_quit.xml
+- recog/xml/smtp_rcptto.xml
+- recog/xml/smtp_rset.xml
+- recog/xml/smtp_turn.xml
+- recog/xml/smtp_vrfy.xml
+- recog/xml/snmp_sysdescr.xml
+- recog/xml/snmp_sysobjid.xml
+- recog/xml/ssh_banners.xml
+- recog/xml/telnet_banners.xml
+- recog/xml/tls_jarm.xml
+- recog/xml/x11_banners.xml
+- recog/xml/x509_issuers.xml
+- recog/xml/x509_subjects.xml
 - spec/data/best_os_match_1.yml
 - spec/data/best_os_match_2.yml
 - spec/data/best_service_match_1.yml
+- spec/data/external_example_fingerprint.xml
+- spec/data/external_example_fingerprint/hp_printer_ex_01.txt
+- spec/data/external_example_fingerprint/hp_printer_ex_02.txt
+- spec/data/external_example_illegal_path_fingerprint.xml
 - spec/data/smb_native_os.txt
 - spec/data/test_fingerprints.xml
 - spec/data/verification_fingerprints.xml
 - spec/data/whitespaced_fingerprint.xml
-- spec/lib/fingerprint_self_test_spec.rb
 - spec/lib/recog/db_spec.rb
 - spec/lib/recog/fingerprint/regexp_factory_spec.rb
 - spec/lib/recog/fingerprint_spec.rb
@@ -207,57 +197,7 @@ files:
 - spec/lib/recog/nizer_spec.rb
 - spec/lib/recog/verify_reporter_spec.rb
 - spec/spec_helper.rb
-- update_cpes.py
-- xml/apache_modules.xml
-- xml/apache_os.xml
-- xml/architecture.xml
-- xml/dns_versionbind.xml
-- xml/favicons.xml
-- xml/fingerprints.xsd
-- xml/ftp_banners.xml
-- xml/h323_callresp.xml
-- xml/hp_pjl_id.xml
-- xml/html_title.xml
-- xml/http_cookies.xml
-- xml/http_servers.xml
-- xml/http_wwwauth.xml
-- xml/imap_banners.xml
-- xml/ldap_searchresult.xml
-- xml/mdns_device-info_txt.xml
-- xml/mdns_workstation_txt.xml
-- xml/mysql_banners.xml
-- xml/mysql_error.xml
-- xml/nntp_banners.xml
-- xml/ntp_banners.xml
-- xml/operating_system.xml
-- xml/pop_banners.xml
-- xml/rsh_resp.xml
-- xml/rtsp_servers.xml
-- xml/sip_banners.xml
-- xml/sip_user_agents.xml
-- xml/smb_native_lm.xml
-- xml/smb_native_os.xml
-- xml/smtp_banners.xml
-- xml/smtp_debug.xml
-- xml/smtp_ehlo.xml
-- xml/smtp_expn.xml
-- xml/smtp_help.xml
-- xml/smtp_mailfrom.xml
-- xml/smtp_noop.xml
-- xml/smtp_quit.xml
-- xml/smtp_rcptto.xml
-- xml/smtp_rset.xml
-- xml/smtp_turn.xml
-- xml/smtp_vrfy.xml
-- xml/snmp_sysdescr.xml
-- xml/snmp_sysobjid.xml
-- xml/ssh_banners.xml
-- xml/telnet_banners.xml
-- xml/tls_jarm.xml
-- xml/x11_banners.xml
-- xml/x509_issuers.xml
-- xml/x509_subjects.xml
-homepage: https://www.github.com/rapid7/recog
+homepage: https://www.github.com/rapid7/recog-ruby
 licenses: []
 metadata: {}
 post_install_message:
@@ -275,31 +215,22 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Network service fingerprint database, classes, and utilities
 test_files:
-- features/data/failing_banners_fingerprints.xml
-- features/data/matching_banners_fingerprints.xml
-- features/data/multiple_banners_fingerprints.xml
-- features/data/no_tests.xml
-- features/data/sample_banner.txt
-- features/data/successful_tests.xml
-- features/data/tests_with_failures.xml
-- features/data/tests_with_warnings.xml
-- features/match.feature
-- features/support/aruba.rb
-- features/support/env.rb
-- features/verify.feature
 - spec/data/best_os_match_1.yml
 - spec/data/best_os_match_2.yml
 - spec/data/best_service_match_1.yml
+- spec/data/external_example_fingerprint.xml
+- spec/data/external_example_fingerprint/hp_printer_ex_01.txt
+- spec/data/external_example_fingerprint/hp_printer_ex_02.txt
+- spec/data/external_example_illegal_path_fingerprint.xml
 - spec/data/smb_native_os.txt
 - spec/data/test_fingerprints.xml
 - spec/data/verification_fingerprints.xml
 - spec/data/whitespaced_fingerprint.xml
-- spec/lib/fingerprint_self_test_spec.rb
 - spec/lib/recog/db_spec.rb
 - spec/lib/recog/fingerprint/regexp_factory_spec.rb
 - spec/lib/recog/fingerprint_spec.rb

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,37 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-1. Do this...
-2. Do that...
-3. Then something happens...
-
-Code that reproduces the behavior: <!-- This is optional -->
-```ruby
-# paste code here, or create a gist, or link to public code snippet
-```
-
-Matcher that reproduces the behavior: <!-- This is optional -->
-```xml
-<!-- paste matcher xml here, or create a gist, or link to public code snippet -->
-```
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-
-**Environment (please complete the following information):**
- - Operating System: 
- - Ruby Version: 
- - Recog Version: 
-
-**Additional context**
-Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,17 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

data/.github/ISSUE_TEMPLATE/fingerprint_request.md

@@ -1,27 +0,0 @@
----
-name: Fingerprint request
-about: Request new fingerprint coverage
-
----
-
-**Request for new fingerprint(s) for a product**  
-Briefly describe the product to be fingerprinted, including vendor and version information.  
-<!-- Include links to relevant public documentation from the vendor or other sources, if available -->
-
-**Provide details about the product**  
-What protocol(s) can be used to retrieve identifiable information about the product?  
-<!-- Examples: Telnet, SSH, SNMP, SMTP, HTTP(S), Vendor Proprietary Protocol -->
-
-What information can be collected on each protocol?  
-<!-- Example: HTTP Server banner: nginx/0.8.53 -->
-
-What request, command, and/or payload can be used to retrieve information on each protocol?  
-<!-- Use code fences like the below example to preserve formatting -->
-```
-HTTP HEAD /
-
-HTTP GET /info.php
-```
-
-**Example banner(s) with specific version info**  
-<!-- Paste raw text here, using code fences to preserve formatting if needed-->

data/.github/PULL_REQUEST_TEMPLATE

@@ -1,24 +0,0 @@
-## Description
-A detailed description of your changes.
-
-
-## Motivation and Context
-Explanation of why these changes are being proposed, including any links to other relevant issues or pull requests.
-
-
-## How Has This Been Tested?
-A clear and concise description of your changes were tested.
-
-
-## Types of changes
-<!--- What types of changes does your code introduce? Remove any that do not apply: -->
-- Bug fix (non-breaking change which fixes an issue)
-- New feature (non-breaking change which adds functionality)
-- Breaking change (fix or feature that would cause existing functionality to change)
-
-
-## Checklist:
-<!--- After submitting the PR, check all of the boxes that apply. -->
-- [ ] I have updated the documentation accordingly (or changes are not required).
-- [ ] I have added tests to cover my changes (or new tests are not required).
-- [ ] All new and existing tests passed.

data/.github/SECURITY.md

@@ -1,35 +0,0 @@
-# Reporting security issues
-
-Thanks for your interest in making Recog more secure! If you feel
-that you have found a security issue involving Metasploit, Meterpreter,
-Recog, or any other Rapid7 open source project, you are welcome to let
-us know in the way that's most comfortable for you.
-
-## Via ZenDesk
-
-You can click on the big blue button at [Rapid7's Vulnerability
-Disclosure][r7-vulns] page, which will get you to our general
-vulnerability reporting system. While this does require a (free) ZenDesk
-account to use, you'll get regular updates on your issue as our software
-support teams work through it. As it happens [that page][r7-vulns] also
-will tell you what to expect when it comes to reporting vulns, how fast
-we'll fix and respond, and all the rest, so it's a pretty good read
-regardless.
-
-## Via email
-
-If you're more of a traditionalist, you can email your finding to
-security@rapid7.com. If you like, you can use our [PGP key][pgp] to
-encrypt your messages, but we certainly don't mind cleartext reports
-over email.
-
-## NOT via GitHub Issues
-
-Please don't! Disclosing security vulnerabilities to public bug trackers
-is kind of mean, even when it's well-intentioned, since you end up
-dropping 0-day on pretty much everyone right out of the gate. We'd prefer
-you didn't!
-
-[r7-vulns]:https://www.rapid7.com/security/disclosure/
-[pgp]:https://keybase.io/rapid7/pgp_keys.asc?fingerprint=9a90aea0576cbcafa39c502ba5e16807959d3eda
-

data/.github/workflows/ci.yml

@@ -1,26 +0,0 @@
-name: CI
-
-on: [push, pull_request]
-
-jobs:
-  test:
-    name: 'Ruby: ${{ matrix.ruby-version }}'
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Run tests
-      run: |
-        bundle exec rake --version
-        bundle exec rake tests
-      env:
-        JRUBY_OPTS: --server -J-Xms512m -J-Xmx2G

data/.gitignore

@@ -1,23 +0,0 @@
-# Ruby and tooling specific
-.yardoc
-coverage/
-doc/
-pkg/
-
-/Gemfile.lock
-
-#Python specific
-venv
-
-# IDE specific
-.vscode/
-.idea
-
-# Misc
-**/.DS_Store
-
-# CPE XML
-official-cpe-dictionary*.xml
-
-# CPE Remap Errors
-errors.txt

data/.rspec

@@ -1,3 +0,0 @@
---color
---warnings
---require spec_helper

data/.ruby-gemset

@@ -1 +0,0 @@
-recog

data/.ruby-version

@@ -1 +0,0 @@
-2.6.6

data/.snyk

@@ -1,10 +0,0 @@
-# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
-python: 3.6.0
-version: v1.14.1
-# ignores vulnerabilities until expiry date; change duration by modifying expiry date
-ignore:
-  SNYK-PYTHON-PYYAML-590151:
-    - pyyaml:
-        reason: Project doesn't use vulnerable code path.
-        expires: 2021-06-01T00:00:00.000Z
-patch: {}

data/.travis.yml

@@ -1,25 +0,0 @@
-language: ruby
-sudo: false
-cache: bundler
-rvm:
-  - '2.5.8'
-  - '2.6.6'
-  - 'jruby-9.1.9.0'
-jdk:
-  - openjdk8
-matrix:
-  allow_failures:
-    - rvm: 'jruby-9.1.9.0'
-before_install:
-  - "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
-  - rake --version
-before_script:
-  - bundle exec rake --version
-script: bundle exec rake tests
-env:
-  global:
-    - CI="travis"
-    - JRUBY_OPTS="--server -J-Xms512m -J-Xmx2G"
-branches:
-  only:
-    - master