recog 2.3.22 → 2.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e33013be558344280c798bb77321b809289a604f55a9f8447bba1cdd2b93151
-  data.tar.gz: 4088c7af5a4dc2250562a610f2bbc6307436fefabe1203f1bd00fd2a9f9c0e3d
+  metadata.gz: a185ae988081b761341e8b8799c23feb5eb2ab85c2da2ce63fffe4f2d7ddcac3
+  data.tar.gz: 89e4ab65e87d190c3ea276d18771edd9c6fc75cb5a0863440be61d165ae5b80c
 SHA512:
-  metadata.gz: abca30fbb5b218e69a2178c0b9b7337af15f1611d67076031ee5d489140505c558fe8981513d8d446bddc91b4166f304b88fed9eb23e22de0d4a5e10f2bfb668
-  data.tar.gz: 57c7e248435b5d52860cd2117a814176673c0e8cb193f854880b3244c91d9b745ee67ca1bc8f2f67f1f3046d26857ca198c81a0e129f9d50b8cc28349070d250
+  metadata.gz: ff83575470bf78d4e5a5c464e0dfeb12ac310ecc856fde6919bf3f7a5e3ba9b69d2be479f9a0cab4cc2c03f7babb6f21673d7015d5cf3182587f86ab3f45f0cf
+  data.tar.gz: 68a7dad6b6cbafc8dac0ac46a0ae7aab6f64c0095c314fa126da107f05b3befff3e4f9f2af41836aa96437746dceeb17718927acc824d842cb9f4a56b234da34

data/.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
+        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.2.20.1', 'jruby']
 
     steps:
     - uses: actions/checkout@v2

data/.github/workflows/verify.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           bundler-cache: true # runs 'bundle install' and caches installed gems automatically
       - name: Run recog verify
-        run: bundle exec recog_verify --no-warnings xml/*.xml
+        run: bundle exec recog_verify --schema-location xml/fingerprints.xsd --no-warnings xml/*.xml
   java-verify:
     name: 'Java Verify'
     runs-on: ubuntu-latest

data/.vscode/bin/monitor-recog-fingerprints.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+ARGS=()
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -s|--schema-location)
+            VALIDATE_SCHEMA="--schema-location $2"
+            shift
+            shift
+            ;;
+        -*|--*)
+            echo "Unknown option $1"
+            exit 1
+            ;;
+        *)
+            ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+set -- "${ARGS[@]}"
+
+if [ $# -eq 0 ]
+then
+    echo "Usage: $(basename $0) [--schema-location SCHEMA_LOCATION] <xml fingerprint directory>"
+    exit 1
+fi
+
+if [ ! -d "$1" ]
+then
+    echo "The XML fingerprint file directory must be supplied."
+    exit 1
+fi
+
+bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
+
+if ! type fswatch &>/dev/null;
+then 
+    echo "'fswatch' is required to monitor fingerprint files for changes and update the editor."
+    echo "See: https://emcrisostomo.github.io/fswatch/ or install with:"
+    echo " MacOS Homebrew: brew install fswatch"
+    echo " Ubuntu/Debian:  apt install fswatch"
+    echo
+    echo "Otherwise, you can re-run this task using the Visual Studio Code command palette"
+    exit 1
+fi
+
+echo "Waiting for changes..."
+fswatch -0 $1 | while read -d "" event; do {
+    echo "Changes detected, validating: ${event}"
+    # TODO: VSCode doesn't support individual/incremental updates to files yet.
+    bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
+    echo "Waiting for changes..."
+}; done

data/.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "redhat.vscode-xml"
+    ]
+}

data/.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+  "xml.fileAssociations": [
+    {
+        "pattern": "xml/*.xml",
+        "systemId": "xml/fingerprints.xsd"
+    }
+ ]
+}

data/.vscode/tasks.json

@@ -0,0 +1,77 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Recog Verify - Background Monitor",
+            "command": ".vscode/bin/monitor-recog-fingerprints.sh",
+            "args": [
+                "--schema-location",
+                "xml/fingerprints.xsd",
+                "${workspaceFolder}/xml"
+            ],
+            "windows": {
+                "command": ""
+            },
+            "type": "process",
+            "isBackground": true,
+            "problemMatcher": {
+                "owner": "recog",
+                "fileLocation": [
+                    "absolute"
+                ],
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
+                    "file": 1,
+                    "severity": 3,
+                    "message": 4,
+                    "location": 2
+                },
+                "background": {
+                    "activeOnStart": true,
+                    "beginsPattern": "^Changes detected",
+                    "endsPattern": "^Waiting for changes"
+                },
+            },
+            "presentation": {
+                "reveal": "always",
+                "revealProblems": "onProblem"
+            },
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        },
+        {
+            "label": "Recog Verify",
+            "command": "bin/recog_verify",
+            "args": [
+                "--schema-location",
+                "xml/fingerprints.xsd",
+                "${workspaceFolder}/xml/*.xml"
+            ],
+            "windows": {
+                "command": ""
+            },
+            "type": "process",
+            "problemMatcher": {
+                "owner": "recog",
+                "fileLocation": [
+                    "absolute"
+                ],
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
+                    "file": 1,
+                    "severity": 3,
+                    "message": 4,
+                    "location": 2
+                }
+            },
+            "presentation": {
+                "reveal": "always",
+                "revealProblems": "onProblem"
+            },
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        }
+    ]
+}

data/CONTRIBUTING.md

@@ -141,6 +141,8 @@ $ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
 MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
 ```
 
+Additionally, in Visual Studio Code, there is a task (.vscode/tasks.json) which will automatically run recog_verify in the background to watch all the XML fingerprint files (under the xml/ subdirectory of this repository). Additionally, if [fswatch](https://github.com/emcrisostomo/fswatch) is installed, whenever XML fingerprint files are added or modified this task will automatically update the Visual Studio Code user interface and highlight any errors or warnings discovered through recog_verify on the correct file/line. You can also manually run the task by bringing up the Visual Studio Code command menu (cmd + shift + P on mac, or ctrl + shift + P for linux/windows) -> Tasks: Run Task -> Recog Verify). Note that in order for the task to run successfully, you must have a valid ruby installed on your PATH with the gems from `bundle install` installed using bundler for that ruby engine. JRuby is not supported as it has issues related to line numbering due to a bug in Nokogiri.
+
 [^back to top](#contributing-to-recog)
 
 

data/bin/recog_verify

@@ -1,12 +1,15 @@
 #!/usr/bin/env ruby
 
 $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'nokogiri'
 require 'optparse'
 require 'ostruct'
 require 'recog'
-require 'recog/verifier_factory'
+require 'recog/formatter'
+require 'recog/verifier'
+require 'recog/verify_reporter'
 
-options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true)
+options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true, schema: nil)
 
 option_parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
@@ -35,6 +38,10 @@ option_parser = OptionParser.new do |opts|
     options.warnings = o
   end
 
+  opts.on("--schema-location SCHEMA_FILE", "Location of the Recog XSD file. If not specified, validation will not be run.") do |schema_file|
+    options.schema = Nokogiri::XML::Schema(File.read(schema_file))
+  end
+
   opts.on("-h", "--help", "Show this message.") do
     puts opts
     exit
@@ -50,13 +57,41 @@ end
 
 warnings = 0
 failures = 0
+formatter = Recog::Formatter.new(options, $stdout)
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
-    ndb = Recog::DB.new(file)
-    verifier = Recog::VerifierFactory.build(options, ndb)
-    verified = verifier.verify
-    failures += verifier.reporter.failure_count
-    warnings += verifier.reporter.warning_count
+    # Create a new reporter per XML file to hold context on success/warn/fails
+    reporter  = Recog::VerifyReporter.new(options, formatter, file)
+
+    begin
+      # Validate the XML database against the recog schema first, if requested
+      if options.schema
+        errors = options.schema.validate(Nokogiri::XML(File.read(file)))
+        if errors.size > 0
+          reporter.report(0) do
+            errors.each do |error|
+              reporter.failure(error.message, error.line)
+            end
+          end
+          # Skip validation of individual fingerprints since the XML itself
+          # is likely malformed.
+          next
+        end
+      end
+
+      # Now read the XML file directly and validate the fingerprints
+      # themselves
+      db = Recog::DB.new(file)
+      verifier = Recog::Verifier.new(db, reporter)
+      verifier.verify
+    rescue Recog::FingerprintParseError => e
+      reporter.failure(e.message, e.line_number)
+    rescue => e
+      reporter.failure(e.message)
+    ensure
+      failures += reporter.failure_count
+      warnings += reporter.warning_count
+    end
   end
 end
 

data/cpe-remap.yaml

@@ -18,6 +18,9 @@ mappings:
       vendor: altn
     aruba_networks:
       vendor: arubanetworks
+    atlassian:
+      products:
+        confluence: confluence_server
     bea:
       products:
         weblogic: weblogic_server
@@ -127,6 +130,8 @@ mappings:
       vendor: mortbay
     munin:
       vendor: munin-monitoring
+    nginx:
+      vendor: f5
     nlnet_labs:
       vendor: nlnetlabs
       products:
@@ -201,6 +206,12 @@ mappings:
     x.org:
       products:
         x.org_x11: x11
+    xiongmai_technology:
+      vendor: xiongmaitech
+    zaphoyd_studios:
+      vendor: zaphoyd
+      products:
+        websocket++: websocketpp
 
   # The following section contains CPE operating system or 'o' remappings. These will
   # ONLY be used for mapping Recog 'os' attributes.
@@ -225,6 +236,7 @@ mappings:
     cisco:
       products:
         adaptive_security_appliance: adaptive_security_appliance_software
+        mds_9000: mds_9000_san-os
         nam: network_analysis_module_software
         pix: pix_firewall_software
         telepresence: telepresence_video_communication_server_software
@@ -243,6 +255,10 @@ mappings:
     debian:
       products:
         linux: debian_linux
+    eltex:
+      vendor: eltex-co
+    fedora_project:
+      vendor: fedoraproject
     hp:
       products:
         ilo: integrated_lights-out_firmware
@@ -286,8 +302,6 @@ mappings:
       vendor: phoenixcontact
     red_hat:
       vendor: redhat
-      products:
-        fedora_core_linux: fedora_core
     software_house:
       vendor: swhouse
     sun:
@@ -306,6 +320,8 @@ mappings:
         vmware_esxi_server: esxi
     wind_river:
       vendor: windriver
+    xiongmai_technology:
+      vendor: xiongmaitech
 
   # The following section contains CPE hardware or 'h' remappings. These will
   # ONLY be used for mapping Recog 'hw' attributes.
@@ -336,6 +352,8 @@ mappings:
     citrix:
       products:
         netscaler_sdx_gateway: netscaler_sdx
+    eltex:
+      vendor: eltex-co
     emc:
       products:
         celerra: celerra_network_attached_storage

data/features/data/schema_failure.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<fingerprints>
+    <fingerprint name="foo"/>
+</fingerprints>

data/features/data/tests_with_failures.xml

@@ -17,4 +17,10 @@
      <param pos="2" name="os.version" />
      <param pos="1" name="os.name" value="Bar" />
    </fingerprint>
+  <fingerprint pattern="^(\S+) ([\d.]+)$">
+    <description>example with untested parameter</description>
+    <!-- Fail: missing example test os.version parameter -->
+    <example>bar 1.0</example>
+    <param pos="1" name="os.version" />
+  </fingerprint>
 </fingerprints>

data/features/support/hooks.rb

@@ -0,0 +1,9 @@
+require "cucumber/platform"
+
+Before "@requires-ruby-platform-java" do
+ skip_this_scenario unless Cucumber::JRUBY
+end
+  
+Before "@unsupported-on-platform-java" do
+ skip_this_scenario if Cucumber::JRUBY
+end

data/features/verify.feature

@@ -16,33 +16,97 @@ Feature: Verify
       """
 
   @no-clobber
-  Scenario: Tests with warnings, warnings enabled
+  Scenario: Tests with warnings, warnings disabled
+    When I run `recog_verify --no-warnings tests_with_warnings.xml`
+    Then it should pass with exactly:
+      """
+      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
+      """
+ 
+  # These tests vary due to line numbering issues in Nokogiri, so there are different versions
+  # of the same test depending on the ruby runtime. Nokogiri uses libxml under CRuby, and a custom
+  # java-based parser under JRuby. The Java-based parser only approximates line numbers, which are
+  # typically off if XML prolog or comments are present.
+  #
+  # See https://github.com/sparklemotion/nokogiri/issues/2380
+  @no-clobber
+  @unsupported-on-platform-java
+  Scenario: Tests with warnings, warnings enabled (CRuby)
     When I run `recog_verify tests_with_warnings.xml`
     Then it should fail with:
       """
-      tests_with_warnings.xml: WARN: 'Pure-FTPd' has no test cases
-      tests_with_warnings.xml: WARN: 'Pure-FTPd' is missing an example that checks for parameter 'pureftpd.config' which is derived from a capture group
-      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 2 warnings, and 0 failures
+      tests_with_warnings.xml:10: WARN: 'Pure-FTPd' has no test cases
+      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 1 warnings, and 0 failures
       """
-    And the exit status should be 2
-
+    And the exit status should be 1
+ 
+  # JRuby 9.2.20.1 and 9.3.3.0 differ in how they parse XML, where the latter is more close to libxml
+  # and Nokogiri. We use a regex test to match results from both versions.
   @no-clobber
-  Scenario: Tests with warnings, warnings disabled
-    When I run `recog_verify --no-warnings tests_with_warnings.xml`
-    Then it should pass with exactly:
+  @requires-ruby-platform-java
+  Scenario: Tests with warnings, warnings enabled (JRuby)
+    When I run `recog_verify tests_with_warnings.xml`
+    Then it should fail with regex:
       """
-      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
+      tests_with_warnings.xml:\d+: WARN: 'Pure-FTPd' has no test cases
+      tests_with_warnings.xml: SUMMARY: Test completed with 1 successful, 1 warnings, and 0 failures
       """
+    And the exit status should be 1
 
   @no-clobber
-  Scenario: Tests with failures
+  @unsupported-on-platform-java
+  Scenario: Tests with failures (CRuby)
     When I run `recog_verify tests_with_failures.xml`
     Then it should fail with:
       """
-      tests_with_failures.xml: FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
-      tests_with_failures.xml: FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
-      tests_with_failures.xml: FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
-      tests_with_failures.xml: FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
-      tests_with_failures.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 4 failures
+      tests_with_failures.xml:3: FAIL: 'foo test' failed to match "bar" with (?-mix:^foo$)'
+      tests_with_failures.xml:8: FAIL: '' failed to match "This almost matches" with (?-mix:^This matches$)'
+      tests_with_failures.xml:13: FAIL: 'bar test's os.name is a non-zero pos but specifies a value of 'Bar'
+      tests_with_failures.xml:13: FAIL: 'bar test' failed to find expected capture group os.version '5.0'. Result was 1.0
+      tests_with_failures.xml:20: FAIL: 'example with untested parameter' is missing an example that checks for parameter 'os.version' which is derived from a capture group
+      tests_with_failures.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 5 failures
+      """
+    And the exit status should be 5
+ 
+  # JRuby 9.2.20.1 and 9.3.3.0 differ in how they parse XML, where the latter is more close to libxml
+  # and Nokogiri. We use a regex test to match results from both versions.
+  @no-clobber
+  @requires-ruby-platform-java
+  Scenario: Tests with failures (JRuby)
+    When I run `recog_verify tests_with_failures.xml`
+    Then it should fail with regex:
+      """
+      tests_with_failures.xml:\d+: FAIL: 'foo test' failed to match "bar" with \(\?-mix:\^foo\$\)'
+      tests_with_failures.xml:\d+: FAIL: '' failed to match "This almost matches" with \(\?-mix:\^This matches\$\)'
+      tests_with_failures.xml:\d+: FAIL: 'bar test's os\.name is a non-zero pos but specifies a value of 'Bar'
+      tests_with_failures.xml:\d+: FAIL: 'bar test' failed to find expected capture group os\.version '5\.0'. Result was 1\.0
+      tests_with_failures.xml:\d+: FAIL: 'example with untested parameter' is missing an example that checks for parameter 'os\.version' which is derived from a capture group
+      tests_with_failures.xml: SUMMARY: Test completed with 1 successful, 0 warnings, and 5 failures
+      """
+    And the exit status should be 5
+
+  @no-clobber
+  @unsupported-on-platform-java
+  Scenario: recog_verify produces XML errors from the XSD with a malformed XML document (CRuby)
+    When I run `recog_verify --schema-location ../../xml/fingerprints.xsd schema_failure.xml`
+    Then it should fail with:
+      """
+      schema_failure.xml:3: FAIL: 3:0: ERROR: Element 'fingerprint', attribute 'name': The attribute 'name' is not allowed.
+      schema_failure.xml:3: FAIL: 3:0: ERROR: Element 'fingerprint': The attribute 'pattern' is required but missing.
+      schema_failure.xml:3: FAIL: 3:0: ERROR: Element 'fingerprint': Missing child element(s). Expected is ( description ).
+      schema_failure.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 3 failures
+      """
+    And the exit status should be 3
+
+  @no-clobber
+  @requires-ruby-platform-java
+  Scenario: recog_verify produces XML errors from the XSD with a malformed XML document (JRuby)
+    When I run `recog_verify --schema_location ../../xml/fingerprints.xsd schema_failure.xml`
+    Then it should fail with:
+      """
+      schema_failure.xml:-1: FAIL: -1:-1: ERROR: cvc-complex-type.3.2.2: Attribute 'name' is not allowed to appear in element 'fingerprint'.
+      schema_failure.xml:-1: FAIL: -1:-1: ERROR: cvc-complex-type.4: Attribute 'pattern' must appear on element 'fingerprint'.
+      schema_failure.xml:-1: FAIL: -1:-1: ERROR: cvc-complex-type.2.4.b: The content of element 'fingerprint' is not complete. One of '{description}' is expected.
+      schema_failure.xml: SUMMARY: Test completed with 0 successful, 0 warnings, and 3 failures
       """
-    And the exit status should be 4
+    And the exit status should be 3

data/identifiers/hw_device.txt

@@ -25,6 +25,7 @@ Handheld Scanner
 Hypervisor
 IP Camera
 IPS
+IPTV
 Industrial Control
 JTAG Adapter
 KVM
@@ -73,6 +74,7 @@ Video Conferencing
 Video Decoder
 Video Encoder
 VoIP
+VoIP Gateway
 VoIP Server
 VoIP Switch
 Voice Appliance

data/identifiers/hw_product.txt

@@ -227,6 +227,8 @@ NAS4Free
 NFVIS
 NPort
 NR900
+NTP-2
+NTP-RG-1402G
 NetScaler Gateway
 NetScaler SDX Gateway
 NetScreen

data/identifiers/os_device.txt

@@ -23,6 +23,7 @@ Hypervisor
 IDS
 IP Camera
 IPS
+IPTV
 KVM
 Lights Out Management
 Linux
@@ -67,6 +68,7 @@ VPN
 Video Conferencing
 ViewStation
 VoIP
+VoIP Gateway
 VoIP Switch
 WAN Accelerator
 WAP

data/identifiers/os_family.txt

@@ -207,6 +207,7 @@ VRP
 Vantage
 VarioLink
 VarioPrint
+Vigor
 VoIP
 VxWorks
 WAP

data/identifiers/os_product.txt

@@ -81,7 +81,7 @@ Fabric OS
 Fastmark M5
 FaxPress
 Fedora
-Fedora Core Linux
+Fedora Core
 Fermentrack
 Fiery Print Server
 Firepower
@@ -91,6 +91,7 @@ FortiOS
 FreeBSD
 FreeNAS Firmware
 Freebox OS
+FreshTomato
 G2 Console Switch
 GAiA OS
 GXP1610 Firmware
@@ -152,6 +153,8 @@ NET+OS
 NFVIS
 NRG Printer
 NT
+NTP-2 Firmware
+NTP-RG-1402G Firmware
 NX-OS
 NetBSD
 NetCache
@@ -222,6 +225,7 @@ SCO UNIX
 SEHI
 SINIX
 SL2100 Firmware
+SMG Firmware
 SPA112 Firmware
 SPA122 Firmware
 SV8100 Firmware
@@ -231,6 +235,7 @@ ScanFront
 ScreenOS
 Secure Linux
 SecureOS
+Serenity
 SmartEdge OS
 SmartServer
 SmoothWall
@@ -258,6 +263,8 @@ Tenable Core
 TetrationOS
 TimOS
 Time Capsule Firmware
+Tomato
+TomatoUSB
 Traverse
 Tru64 Unix
 UCM6202 Firmware