recog 2.3.20 → 2.3.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb4df95cbe1561c384b06be8c36fcea1e51df3c6cdb86a2a944715213d119ae8
-  data.tar.gz: 751fa73b20c6fb9f1c372be5503c07302101b77d729cdce3befee2981651f1ca
+  metadata.gz: a185ae988081b761341e8b8799c23feb5eb2ab85c2da2ce63fffe4f2d7ddcac3
+  data.tar.gz: 89e4ab65e87d190c3ea276d18771edd9c6fc75cb5a0863440be61d165ae5b80c
 SHA512:
-  metadata.gz: 6612cf0d0c5f19cd1a913123fe3c4fce9772ac82b7a07f78ace94f51b1681210a8dfacde3624b2b54c7d66f2f1530771d9a592c3d5bda8bde897d4f9713c2ef9
-  data.tar.gz: 7b935f573b7b4050b2b06e2b8965af9201bb385e2695f9859e6ecf233f93aadb9331d648b18a506528efbd97e2821d0b9816970bfd5df978262b05ec8aeb9f8e
+  metadata.gz: ff83575470bf78d4e5a5c464e0dfeb12ac310ecc856fde6919bf3f7a5e3ba9b69d2be479f9a0cab4cc2c03f7babb6f21673d7015d5cf3182587f86ab3f45f0cf
+  data.tar.gz: 68a7dad6b6cbafc8dac0ac46a0ae7aab6f64c0095c314fa126da107f05b3befff3e4f9f2af41836aa96437746dceeb17718927acc824d842cb9f4a56b234da34

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "11:00"
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.1.17.0', 'jruby']
+        ruby-version: ['2.5', '2.6', '2.7', '3.0', 'jruby-9.2.20.1', 'jruby']
 
     steps:
     - uses: actions/checkout@v2

data/.github/workflows/verify.yml

@@ -0,0 +1,89 @@
+name: Verify
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'xml/**.xml'
+  pull_request:
+    paths:
+      - 'xml/**.xml'
+
+jobs:
+  standardize:
+    name: 'Standardize'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog standardize
+        run: bundle exec bin/recog_standardize xml/*.xml
+  ruby-verify:
+    name: 'Ruby Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Ruby implementation
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run recog verify
+        run: bundle exec recog_verify --schema-location xml/fingerprints.xsd --no-warnings xml/*.xml
+  java-verify:
+    name: 'Java Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Java implementation
+        uses: actions/checkout@v2
+        with:
+          repository: rapid7/recog-java
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: '17'
+      - name: Cache Maven packages
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2
+      - name: Build with Maven
+        run: mvn --batch-mode --no-transfer-progress install -Dmaven.antrun.skip=true -DskipTests
+      - name: Run recog verify
+        run: mvn --batch-mode --no-transfer-progress --projects recog-verify exec:java -Dexec.mainClass="com.rapid7.recog.verify.RecogVerifier" -Dexec.args="--no-warnings recog-content/xml/*.xml"
+  go-verify:
+    name: 'Go Verify'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Checkout Go implementation
+        uses: actions/checkout@v2
+        with:
+          repository: RumbleDiscovery/recog-go
+      - name: Checkout recog content
+        uses: actions/checkout@v2
+        with:
+          path: recog-content
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.1'
+      - name: Run recog verify
+        run: go run cmd/recog_verify/main.go recog-content/xml/

data/.vscode/bin/monitor-recog-fingerprints.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+ARGS=()
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -s|--schema-location)
+            VALIDATE_SCHEMA="--schema-location $2"
+            shift
+            shift
+            ;;
+        -*|--*)
+            echo "Unknown option $1"
+            exit 1
+            ;;
+        *)
+            ARGS+=("$1")
+            shift
+            ;;
+    esac
+done
+set -- "${ARGS[@]}"
+
+if [ $# -eq 0 ]
+then
+    echo "Usage: $(basename $0) [--schema-location SCHEMA_LOCATION] <xml fingerprint directory>"
+    exit 1
+fi
+
+if [ ! -d "$1" ]
+then
+    echo "The XML fingerprint file directory must be supplied."
+    exit 1
+fi
+
+bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
+
+if ! type fswatch &>/dev/null;
+then 
+    echo "'fswatch' is required to monitor fingerprint files for changes and update the editor."
+    echo "See: https://emcrisostomo.github.io/fswatch/ or install with:"
+    echo " MacOS Homebrew: brew install fswatch"
+    echo " Ubuntu/Debian:  apt install fswatch"
+    echo
+    echo "Otherwise, you can re-run this task using the Visual Studio Code command palette"
+    exit 1
+fi
+
+echo "Waiting for changes..."
+fswatch -0 $1 | while read -d "" event; do {
+    echo "Changes detected, validating: ${event}"
+    # TODO: VSCode doesn't support individual/incremental updates to files yet.
+    bin/recog_verify $VALIDATE_SCHEMA "$1/*.xml"
+    echo "Waiting for changes..."
+}; done

data/.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "redhat.vscode-xml"
+    ]
+}

data/.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+  "xml.fileAssociations": [
+    {
+        "pattern": "xml/*.xml",
+        "systemId": "xml/fingerprints.xsd"
+    }
+ ]
+}

data/.vscode/tasks.json

@@ -0,0 +1,77 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "Recog Verify - Background Monitor",
+            "command": ".vscode/bin/monitor-recog-fingerprints.sh",
+            "args": [
+                "--schema-location",
+                "xml/fingerprints.xsd",
+                "${workspaceFolder}/xml"
+            ],
+            "windows": {
+                "command": ""
+            },
+            "type": "process",
+            "isBackground": true,
+            "problemMatcher": {
+                "owner": "recog",
+                "fileLocation": [
+                    "absolute"
+                ],
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
+                    "file": 1,
+                    "severity": 3,
+                    "message": 4,
+                    "location": 2
+                },
+                "background": {
+                    "activeOnStart": true,
+                    "beginsPattern": "^Changes detected",
+                    "endsPattern": "^Waiting for changes"
+                },
+            },
+            "presentation": {
+                "reveal": "always",
+                "revealProblems": "onProblem"
+            },
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        },
+        {
+            "label": "Recog Verify",
+            "command": "bin/recog_verify",
+            "args": [
+                "--schema-location",
+                "xml/fingerprints.xsd",
+                "${workspaceFolder}/xml/*.xml"
+            ],
+            "windows": {
+                "command": ""
+            },
+            "type": "process",
+            "problemMatcher": {
+                "owner": "recog",
+                "fileLocation": [
+                    "absolute"
+                ],
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):\\s+(WARN|FAIL):\\s+(.*)$",
+                    "file": 1,
+                    "severity": 3,
+                    "message": 4,
+                    "location": 2
+                }
+            },
+            "presentation": {
+                "reveal": "always",
+                "revealProblems": "onProblem"
+            },
+            "runOptions": {
+                "runOn": "folderOpen"
+            }
+        }
+    ]
+}

data/CONTRIBUTING.md

@@ -74,6 +74,12 @@ Generally, this should only need to be done once, or if you need to start over.
     git fetch --all
     ```
 
+1. Set up git hooks to help identify potential issues with your contributions:
+
+    ```bash
+    ln -sf ../../tools/dev/hooks/pre-commit .git/hooks/pre-commit
+    ```
+
 [^back to top](#contributing-to-recog)
 
 ### Branch and Improve
@@ -135,6 +141,8 @@ $ echo 'OpenSSH_6.6p1 Ubuntu-2ubuntu1' | bin/recog_match xml/ssh_banners.xml -
 MATCH: {"matched"=>"OpenSSH running on Ubuntu 14.04", "service.version"=>"6.6p1", "openssh.comment"=>"Ubuntu-2ubuntu1", "service.vendor"=>"OpenBSD", "service.family"=>"OpenSSH", "service.product"=>"OpenSSH", "os.vendor"=>"Ubuntu", "os.device"=>"General", "os.family"=>"Linux", "os.product"=>"Linux", "os.version"=>"14.04", "service.protocol"=>"ssh", "fingerprint_db"=>"ssh.banner", "data"=>"OpenSSH_6.6p1 Ubuntu-2ubuntu1"}
 ```
 
+Additionally, in Visual Studio Code, there is a task (.vscode/tasks.json) which will automatically run recog_verify in the background to watch all the XML fingerprint files (under the xml/ subdirectory of this repository). Additionally, if [fswatch](https://github.com/emcrisostomo/fswatch) is installed, whenever XML fingerprint files are added or modified this task will automatically update the Visual Studio Code user interface and highlight any errors or warnings discovered through recog_verify on the correct file/line. You can also manually run the task by bringing up the Visual Studio Code command menu (cmd + shift + P on mac, or ctrl + shift + P for linux/windows) -> Tasks: Run Task -> Recog Verify). Note that in order for the task to run successfully, you must have a valid ruby installed on your PATH with the gems from `bundle install` installed using bundler for that ruby engine. JRuby is not supported as it has issues related to line numbering due to a bug in Nokogiri.
+
 [^back to top](#contributing-to-recog)
 
 

data/README.md

@@ -76,6 +76,23 @@ The `example` string can be base64 encoded to permit the use of unprintable char
 </example>
 ````
 
+Additionally, examples can be placed in a directory with the same base name as the XML file, in the same directory as the XML file:
+
+```
+xml/services.xml
+xml/services/file1
+xml/services/file2
+...
+```
+
+They can then be loaded using the `_filename` attribute:
+
+```xml
+<example _filename="file1"/>
+```
+
+This is useful for long examples.
+
 [^back to top](#recog-a-recognition-framework)
 
 ## Contributing

data/bin/recog_standardize

@@ -61,6 +61,7 @@ hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
 svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
 svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
 
+missing_count = 0
 
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
@@ -70,6 +71,7 @@ ARGV.each do |arg|
         paramIndex, val = v
         if ! fields[k]
           puts "FIELD MISSING: #{k}"
+          missing_count += 1
           fields[k] = true
         end
         next if paramIndex != 0
@@ -79,51 +81,61 @@ ARGV.each do |arg|
         when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
           if ! vendors[val]
             puts "VENDOR MISSING: #{val}"
+            missing_count += 1
             vendors[val] = true
           end
         when "os.arch"
           if ! os_arch[val]
             puts "OS ARCH MISSING: #{val}"
+            missing_count += 1
             os_arch[val] = true
           end          
         when "os.product"
           if ! os_prod[val]
             puts "OS PRODUCT MISSING: #{val}"
+            missing_count += 1
             os_prod[val] = true
           end
         when "os.family"
           if ! os_family[val]
             puts "OS FAMILY MISSING: #{val}"
+            missing_count += 1
             os_family[val] = true
           end
         when "os.device"
           if ! os_device[val]
             puts "OS DEVICE MISSING: #{val}"
+            missing_count += 1
             os_device[val] = true
           end
         when "hw.product"
           if ! hw_prod[val]
             puts "HW PRODUCT MISSING: #{val}"
+            missing_count += 1
             hw_prod[val] = true
           end
         when "hw.family"
           if ! hw_family[val]
             puts "HW FAMILY MISSING: #{val}"
+            missing_count += 1
             hw_family[val] = true
           end
         when "hw.device"
           if ! hw_device[val]
             puts "HW DEVICE MISSING: #{val}"
+            missing_count += 1
             hw_device[val] = true
           end          
         when "service.product", "service.component.product"
           if ! svc_prod[val]
             puts "SERVICE PRODUCT MISSING: #{val}"
+            missing_count += 1
             svc_prod[val] = true
           end            
         when "service.family"
           if ! svc_family[val]
             puts "SERVICE FAMILY MISSING: #{val}"
+            missing_count += 1
             svc_family[val] = true
           end          
         end
@@ -132,17 +144,20 @@ ARGV.each do |arg|
   end
 end
 
-exit if ! options.write
+if options.write
+  # Write back the unique identifiers
+  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+  write_identifiers(fields, File.join(bdir, "fields.txt"))
+  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+  write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+  write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
+  write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
+  write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
+  write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+  write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+end
 
-# Write back the unique identifiers
-write_identifiers(vendors, File.join(bdir, "vendor.txt"))
-write_identifiers(fields, File.join(bdir, "fields.txt"))
-write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
-write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
-write_identifiers(os_family, File.join(bdir, "os_family.txt"))
-write_identifiers(os_device, File.join(bdir, "os_device.txt"))
-write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
-write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
-write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
-write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
-write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
+exit_code = (missing_count > 0 ? 1 : 0)
+exit(exit_code)

data/bin/recog_verify

@@ -1,12 +1,15 @@
 #!/usr/bin/env ruby
 
 $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'nokogiri'
 require 'optparse'
 require 'ostruct'
 require 'recog'
-require 'recog/verifier_factory'
+require 'recog/formatter'
+require 'recog/verifier'
+require 'recog/verify_reporter'
 
-options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true)
+options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true, schema: nil)
 
 option_parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
@@ -35,6 +38,10 @@ option_parser = OptionParser.new do |opts|
     options.warnings = o
   end
 
+  opts.on("--schema-location SCHEMA_FILE", "Location of the Recog XSD file. If not specified, validation will not be run.") do |schema_file|
+    options.schema = Nokogiri::XML::Schema(File.read(schema_file))
+  end
+
   opts.on("-h", "--help", "Show this message.") do
     puts opts
     exit
@@ -50,14 +57,41 @@ end
 
 warnings = 0
 failures = 0
+formatter = Recog::Formatter.new(options, $stdout)
 ARGV.each do |arg|
   Dir.glob(arg).each do |file|
-    ndb = Recog::DB.new(file)
-    options.fingerprints = ndb.fingerprints
-    verifier = Recog::VerifierFactory.build(options)
-    verified = verifier.verify
-    failures += verifier.reporter.failure_count
-    warnings += verifier.reporter.warning_count
+    # Create a new reporter per XML file to hold context on success/warn/fails
+    reporter  = Recog::VerifyReporter.new(options, formatter, file)
+
+    begin
+      # Validate the XML database against the recog schema first, if requested
+      if options.schema
+        errors = options.schema.validate(Nokogiri::XML(File.read(file)))
+        if errors.size > 0
+          reporter.report(0) do
+            errors.each do |error|
+              reporter.failure(error.message, error.line)
+            end
+          end
+          # Skip validation of individual fingerprints since the XML itself
+          # is likely malformed.
+          next
+        end
+      end
+
+      # Now read the XML file directly and validate the fingerprints
+      # themselves
+      db = Recog::DB.new(file)
+      verifier = Recog::Verifier.new(db, reporter)
+      verifier.verify
+    rescue Recog::FingerprintParseError => e
+      reporter.failure(e.message, e.line_number)
+    rescue => e
+      reporter.failure(e.message)
+    ensure
+      failures += reporter.failure_count
+      warnings += reporter.warning_count
+    end
   end
 end
 

data/cpe-remap.yaml

@@ -18,6 +18,9 @@ mappings:
       vendor: altn
     aruba_networks:
       vendor: arubanetworks
+    atlassian:
+      products:
+        confluence: confluence_server
     bea:
       products:
         weblogic: weblogic_server
@@ -127,6 +130,8 @@ mappings:
       vendor: mortbay
     munin:
       vendor: munin-monitoring
+    nginx:
+      vendor: f5
     nlnet_labs:
       vendor: nlnetlabs
       products:
@@ -140,6 +145,8 @@ mappings:
     parallels:
       products:
         plesk: parallels_plesk_panel
+    phoenix_contact:
+      vendor: phoenixcontact
     plesk:
       vendor: parallels
     proftpd_project:
@@ -159,6 +166,8 @@ mappings:
         jboss_eap: jboss_enterprise_application_platform
         jbossweb: jboss_web_framework_kit
         red_hat_directory_server: directory_server
+    rundeck:
+      vendor: pagerduty
     serv-u:
       vendor: solarwinds
     squid_cache:
@@ -180,7 +189,7 @@ mappings:
     tor_project:
       vendor: torproject
     traefik_labs:
-      vendor: containous
+      vendor: traefik
       products:
         traefik_proxy: traefik
     twistedmatrix:
@@ -197,6 +206,12 @@ mappings:
     x.org:
       products:
         x.org_x11: x11
+    xiongmai_technology:
+      vendor: xiongmaitech
+    zaphoyd_studios:
+      vendor: zaphoyd
+      products:
+        websocket++: websocketpp
 
   # The following section contains CPE operating system or 'o' remappings. These will
   # ONLY be used for mapping Recog 'os' attributes.
@@ -208,6 +223,11 @@ mappings:
     apple:
       products:
         ios: iphone_os
+        mac_os: macos
+    brocade:
+      vendor: broadcom
+      products:
+        fabric_os: fabric_operating_system
     centos:
       products:
         linux: centos
@@ -216,6 +236,7 @@ mappings:
     cisco:
       products:
         adaptive_security_appliance: adaptive_security_appliance_software
+        mds_9000: mds_9000_san-os
         nam: network_analysis_module_software
         pix: pix_firewall_software
         telepresence: telepresence_video_communication_server_software
@@ -234,6 +255,10 @@ mappings:
     debian:
       products:
         linux: debian_linux
+    eltex:
+      vendor: eltex-co
+    fedora_project:
+      vendor: fedoraproject
     hp:
       products:
         ilo: integrated_lights-out_firmware
@@ -273,10 +298,12 @@ mappings:
         ilom: integrated_lights_out_manager_firmware
     palo_alto_networks:
       vendor: paloaltonetworks
+    phoenix_contact:
+      vendor: phoenixcontact
     red_hat:
       vendor: redhat
-      products:
-        fedora_core_linux: fedora_core
+    software_house:
+      vendor: swhouse
     sun:
       products:
         solaris: sunos
@@ -293,22 +320,54 @@ mappings:
         vmware_esxi_server: esxi
     wind_river:
       vendor: windriver
+    xiongmai_technology:
+      vendor: xiongmaitech
 
   # The following section contains CPE hardware or 'h' remappings. These will
   # ONLY be used for mapping Recog 'hw' attributes.
   h:
+    apple:
+      products:
+        imac_(retina_4k_21.5-inch_2019): imac
+        imac_(retina_5k_27-inch_2017): imac
+        imac_(retina_5k_27-inch_2019): imac
+        imac_(retina_5k_27-inch_2020): imac
+        macbook_air_(13-inch_2017): macbook_air
+        macbook_air_(m1_2020): macbook_air
+        macbook_air_(retina_13-inch_2018): macbook_air
+        macbook_air_(retina_13-inch_2019): macbook_air
+        macbook_air_(retina_13-inch_2020): macbook_air
+        macbook_pro_(13-inch_2018_four_thunderbolt_3_ports): macbook_pro
+        macbook_pro_(13-inch_2019_two_thunderbolt_3_ports): macbook_pro
+        macbook_pro_(13-inch_2020): macbook_pro
+        macbook_pro_(13-inch_m1_2020): macbook_pro
+        macbook_pro_(15-inch_2018): macbook_pro
+        macbook_pro_(15-inch_2019): macbook_pro
+        macbook_pro_(16-inch_2019): macbook_pro
+        macbook_pro_(retina_13-inch_early_2015): macbook_pro
+        macbook_pro_(retina_15-inch_mid_2015): macbook_pro
     cisco:
       products:
         nam: network_analysis_module
     citrix:
       products:
         netscaler_sdx_gateway: netscaler_sdx
+    eltex:
+      vendor: eltex-co
     emc:
       products:
         celerra: celerra_network_attached_storage
     hp:
       products:
         ilo: integrated_lights-out
+    kace:
+      vendor: dell
+      products:
+        k1000: kace_k1000_systems_management_appliance
+    phoenix_contact:
+      vendor: phoenixcontact
+    software_house:
+      vendor: swhouse
     tandberg:
       vendor: cisco
     ubiquiti:

data/features/data/schema_failure.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<fingerprints>
+    <fingerprint name="foo"/>
+</fingerprints>

data/features/data/tests_with_failures.xml

@@ -17,4 +17,10 @@
      <param pos="2" name="os.version" />
      <param pos="1" name="os.name" value="Bar" />
    </fingerprint>
+  <fingerprint pattern="^(\S+) ([\d.]+)$">
+    <description>example with untested parameter</description>
+    <!-- Fail: missing example test os.version parameter -->
+    <example>bar 1.0</example>
+    <param pos="1" name="os.version" />
+  </fingerprint>
 </fingerprints>