recog 2.3.6 → 2.3.11

data/bin/recog_cleanup

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+
+# Cleanup trailing whitespace around fingerprints
+Dir[ File.expand_path(File.join(File.dirname(__FILE__), "..", "xml")) + "/*.xml" ].each do |f|
+    data = File.read(f).
+        gsub(/\s+$/, '').                           # Trailing whitespace and empty lines
+        gsub("</fingerprint>", "</fingerprint>\n"). # Every fingerprint should have an empty line after it
+        gsub("-->", "-->\n")                        # Every comment should have an empty line after it
+
+    File.write(f, data)
+end

data/bin/recog_standardize

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'recog'
+
+def load_identifiers(path)
+  res = {}
+  File.readlines(path).map{|line| line.strip}.each do |ident|
+    res[ident] = true
+  end
+  return res
+end
+
+def write_identifiers(vals, path)
+  res = []
+  vals.each_pair do |k,v|
+    res = res.push(k)
+  end
+  res = res.map{|x| x.strip}.select{|x| x.length > 0}.sort.uniq
+  File.write(path, res.join("\n") + "\n")
+end
+
+bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
+
+options = OpenStruct.new(write: false)
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
+  opts.separator "Verifies that each fingerprint asserts known identifiers."
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-w", "--write") do
+      options.write = true
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.empty?
+  $stderr.puts 'Missing XML fingerprint files'
+  puts option_parser
+  exit(1)
+end
+
+# Load the unique identifiers
+vendors = load_identifiers(File.join(bdir, "vendor.txt"))
+os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
+os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
+os_family = load_identifiers(File.join(bdir, "os_family.txt"))
+os_device = load_identifiers(File.join(bdir, "os_device.txt"))
+hw_prod = load_identifiers(File.join(bdir, "hw_product.txt"))
+hw_family = load_identifiers(File.join(bdir, "hw_family.txt"))
+hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
+svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
+svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
+
+
+ARGV.each do |arg|
+  Dir.glob(arg).each do |file|
+    ndb = Recog::DB.new(file)
+    ndb.fingerprints.each do |f|
+      f.params.each do |k,v|
+        paramIndex, val = v
+        next if paramIndex != 0
+        next if val.index("{") != -1
+        next if val.strip == ""
+        case k
+        when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
+          if ! vendors[val]
+            puts "VENDOR MISSING: #{val}"
+            vendors[val] = true
+          end
+        when "os.arch"
+          if ! os_arch[val]
+            puts "OS ARCH MISSING: #{val}"
+            os_arch[val] = true
+          end          
+        when "os.product"
+          if ! os_prod[val]
+            puts "OS PRODUCT MISSING: #{val}"
+            os_prod[val] = true
+          end
+        when "os.family"
+          if ! os_family[val]
+            puts "OS FAMILY MISSING: #{val}"
+            os_family[val] = true
+          end
+        when "os.device"
+          if ! os_device[val]
+            puts "OS DEVICE MISSING: #{val}"
+            os_device[val] = true
+          end
+        when "hw.product"
+          if ! hw_prod[val]
+            puts "HW PRODUCT MISSING: #{val}"
+            hw_prod[val] = true
+          end
+        when "hw.family"
+          if ! hw_family[val]
+            puts "HW FAMILY MISSING: #{val}"
+            hw_family[val] = true
+          end
+        when "hw.device"
+          if ! hw_device[val]
+            puts "HW DEVICE MISSING: #{val}"
+            hw_device[val] = true
+          end          
+        when "service.product"
+          if ! svc_prod[val]
+            puts "SERVICE PRODUCT MISSING: #{val}"
+            svc_prod[val] = true
+          end            
+        when "service.family"
+          if ! svc_family[val]
+            puts "SERVICE FAMILY MISSING: #{val}"
+            svc_family[val] = true
+          end          
+        end
+      end
+    end
+  end
+end
+
+exit if ! options.write
+
+# Write back the unique identifiers
+write_identifiers(vendors, File.join(bdir, "vendor.txt"))
+write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
+write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
+write_identifiers(os_family, File.join(bdir, "os_family.txt"))
+write_identifiers(os_device, File.join(bdir, "os_device.txt"))
+write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
+write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
+write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
+write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
+write_identifiers(svc_family, File.join(bdir, "service_family.txt"))

data/cpe-remap.yaml

@@ -1,4 +1,6 @@
 mappings:
+  alpine:
+    vendor: alpinelinux
   apache:
     vendor: apache
     products:
@@ -45,10 +47,16 @@ mappings:
     vendor: ibm
     products:
       lotus_domino: lotus_domino_server
+      os/400: os_400
+  jamf:
+    products:
+      jamf_pro: jamf
   juniper:
     vendor: juniper
     products:
       junos_os: junos
+  kibana:
+    vendor: elasticsearch
   linux:
     vendor: linux
     products:
@@ -94,6 +102,11 @@ mappings:
     vendor: paloaltonetworks
     products:
       pa_firewall: pan-os
+  parallels:
+    products:
+      plesk: parallels_plesk_panel
+  plesk:
+    vendor: parallels
   proftpd_project:
     vendor: proftpd
   realvnc_ltd.:
@@ -113,6 +126,13 @@ mappings:
     vendor: sun
     products:
       solaris: sunos
+  tandberg:
+    vendor: cisco
+  tightvnc:
+    products:
+      desktop: tightvnc
+  ubiquiti:
+    vendor: ui
   ubuntu:
     vendor: canonical
     products:

data/features/match.feature

@@ -1,4 +1,5 @@
 Feature: Match
+  @no-clobber
   Scenario: Finds matches
     When I run `recog_match matching_banners_fingerprints.xml sample_banner.txt`
     Then it should pass with:
@@ -7,6 +8,7 @@ Feature: Match
       MATCH: {"matched"=>"SunOS/Solaris", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "service.protocol"=>"ftp", "fingerprint_db"=>"matching_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
       """
 
+  @no-clobber
   Scenario: Fails at finding matches
     When I run `recog_match failing_banners_fingerprints.xml sample_banner.txt`
     Then it should pass with:
@@ -15,6 +17,7 @@ Feature: Match
       FAIL: polaris FTP server (SunOS 5.8) ready
       """
 
+  @no-clobber
   Scenario: Finds multiple matches
     When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --multi-match`
     Then it should pass with:
@@ -23,6 +26,7 @@ Feature: Match
       MATCHES: {"matched"=>"Generic FTP, Checks for the existence of the word FTP in the line", "service.protocol"=>"", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."},{"matched"=>"SunOS/Solaris", "service.protocol"=>"ftp", "os.vendor"=>"Sun", "os.family"=>"Solaris", "os.product"=>"Solaris", "os.device"=>"General", "host.name"=>"polaris", "os.version"=>"5.8", "fingerprint_db"=>"multiple_banners_fingerprints", "data"=>"polaris FTP server (SunOS 5.8) ready."}
       """
 
+  @no-clobber
   Scenario: Finds first matches using no-multi-match flag
     When I run `recog_match multiple_banners_fingerprints.xml sample_banner.txt --no-multi-match`
     Then it should pass with:

data/features/support/aruba.rb

@@ -0,0 +1,3 @@
+Aruba.configure do |config|
+  config.working_directory = 'features/data'
+end

data/features/verify.feature

@@ -1,4 +1,5 @@
 Feature: Verify
+  @no-clobber
   Scenario: No tests
     When I run `recog_verify no_tests.xml`
     Then it should pass with:
@@ -6,6 +7,7 @@ Feature: Verify
       SUMMARY: Test completed with 0 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Successful tests
     When I run `recog_verify successful_tests.xml`
     Then it should pass with:
@@ -13,6 +15,7 @@ Feature: Verify
       SUMMARY: Test completed with 4 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Tests with warnings, warnings enabled
     When I run `recog_verify tests_with_warnings.xml`
     Then it should fail with:
@@ -23,6 +26,7 @@ Feature: Verify
       """
     And the exit status should be 2
 
+  @no-clobber
   Scenario: Tests with warnings, warnings disabled
     When I run `recog_verify --no-warnings tests_with_warnings.xml`
     Then it should pass with:
@@ -30,6 +34,7 @@ Feature: Verify
       SUMMARY: Test completed with 1 successful, 0 warnings, and 0 failures
       """
 
+  @no-clobber
   Scenario: Tests with failures
     When I run `recog_verify tests_with_failures.xml`
     Then it should fail with:

data/identifiers/README.md

@@ -0,0 +1,56 @@
+# Recog: Identifiers
+
+This directory contains lists of standard identifiers for mapping Recog matches. The goal is define a standard set of constants to represent known software, hardware, vendors, and categories.
+
+This is currently incomplete and will be updated as standardization work moves forward.
+
+Fingerprints should use these identifiers whenever possible; if a different name or syntax for a given identifier is preferred, this should be implemented in the application through a mapping function.
+
+## Lists
+
+### Vendors
+
+`vendor.txt` defines known vendor names, covering services, operating systems, and hardware.
+
+### Operating Systems
+
+`os_architecture.txt` defines known CPU types.
+
+`os_product.txt` defines known operating system names.
+
+`os_family.txt` defines known operating system families.
+
+`os_device.txt` defines known types of devices by function or purpose.
+
+
+### Hardware
+
+`hw_product.txt` defines known hardware product names.
+
+`hw_family.txt` defines known hardware product families.
+
+`hw_device.txt` defines known types of devices by function or purpose (overlaps with `os_device.txt`).
+
+### Services
+
+`service_product.txt` defines known service product names.
+
+`service_family.txt` defines known service product families.
+
+### Software
+
+`software_product.txt` defines known software product names.
+
+`software_family.txt` defines known software product families.
+
+`software_class.txt` defines known types of software by function or purpose.
+
+## Pending Work
+
+  * All existing fingerprints should be correlated against these lists to identify mismatches and updated accordingly.
+
+  * All net new identifiers from the existing fingerprints should be merged into these lists.
+
+  * All fingerprint assertions should be enumerated, documented, and standardized where possible (`host.mac`, etc).
+
+  * Hardware identifiers should be enumerated, consolidated, and standardized.

data/identifiers/hw_device.txt

@@ -0,0 +1,77 @@
+ADSL Modem
+AV Receiver
+Access Control
+Alarm Panel
+Appliance
+Audio Encoder
+Broadband router
+Building Automation
+Cable Modem
+Check Scanner
+DOCSIS Cable Modem
+DSL Modem
+DVR
+Data Terminal
+Desktop
+Device
+Device Hub
+Device Server
+Display Controller
+Environment Control
+Ethernet Adapter
+Firewall
+HMI Controller
+Handheld Scanner
+IP Camera
+IPS
+Industrial Control
+JTAG Adapter
+KVM
+Laptop
+Light Bulb
+Lights Out Management
+Media Receiver
+Media Server
+Mobile Phone
+Monitoring
+Multifunction Device
+NAS
+Network Appliance
+Network Audio
+Network Management Device
+PLC
+Power Relay
+Power device
+Powerline
+Print server
+Printer
+Router
+SD-WAN Appliance
+SIP Device
+SIP Gateway
+Scanner
+Security Appliance
+Smart TV
+Storage
+Storage Appliance
+Support Appliance
+Switch
+Tablet
+Tape Library
+Telecom
+Test Instrument
+VPN
+Video Conference
+Video Conferencing
+Video Decoder
+Video Encoder
+VoIP
+VoIP Server
+VoIP Switch
+Voice Appliance
+WAP
+WLAN Repeater
+Web cam
+Whiteboard
+Wireless Controller
+Wireless Presenter

data/identifiers/hw_family.txt

@@ -0,0 +1,96 @@
+AR Series
+Adaptive Security Appliance
+Aficio
+AirPort
+Apple TV
+Communication Manager
+DVR
+DiskStation
+Document Centre
+Extended Systems ExtendNet
+FRITZ!Box
+FRITZ!Powerline
+FRITZ!WLAN Repeater
+FS
+Firewall-1
+Forms Printer
+FortiGate
+GW25
+GXV
+GXW
+HDX
+HandyTone
+HomePod
+Honeywell
+ILOM
+IMDVR
+ION
+JetDirect
+LaserJet
+LinkCom Xpress
+MGate
+MPEG4 DVR
+MT
+MX Series
+Mac mini
+MacBook
+MacBook Pro
+MegaRAC
+Multifunction
+My Book
+NE
+NPort
+NetVanta
+Network Audio
+Network Video Door Station
+Optra
+Orbi
+POWER System
+Phaser
+Primergy
+Pro Series
+RMX
+ReadyNAS
+RealPresence
+RealPresence Group
+Router
+S500 Range
+SIP Device
+SIP Gateway
+Service Access Switch
+Service Router
+SoundPoint
+SoundTouch
+SpeedTouch
+Speedport
+Storage
+Sun Fire
+Sunny
+Switch
+System X
+TASKalfa
+TelePresence
+Time Capsule
+TippingPoint
+Turbo Station
+UniFi
+Unified Security Gateway
+VSX
+VoIP
+WD2GO
+WiMax
+Wide Format
+Wide Format Printer
+WorkCentre
+WorkCentre Pro
+Xserve
+ZXDSL
+ZXHN
+ZXV
+iLO
+iMac
+iPad
+iPad Air
+iPad Pro
+iPad mini
+iPhone