recog 2.3.22 → 3.0.2

data/bin/recog_standardize

@@ -1,163 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-def load_identifiers(path)
-  res = {}
-  File.readlines(path).map{|line| line.strip}.each do |ident|
-    res[ident] = true
-  end
-  return res
-end
-
-def write_identifiers(vals, path)
-  res = []
-  vals.each_pair do |k,v|
-    res = res.push(k)
-  end
-  res = res.map{|x| x.strip}.select{|x| x.length > 0}.sort.uniq
-  File.write(path, res.join("\n") + "\n")
-end
-
-bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
-
-options = OpenStruct.new(write: false)
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
-  opts.separator "Verifies that each fingerprint asserts known identifiers."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-w", "--write") do
-      options.write = true
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.empty?
-  $stderr.puts 'Missing XML fingerprint files'
-  puts option_parser
-  exit(1)
-end
-
-# Load the unique identifiers
-vendors = load_identifiers(File.join(bdir, "vendor.txt"))
-fields = load_identifiers(File.join(bdir, "fields.txt"))
-os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
-os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
-os_family = load_identifiers(File.join(bdir, "os_family.txt"))
-os_device = load_identifiers(File.join(bdir, "os_device.txt"))
-hw_prod = load_identifiers(File.join(bdir, "hw_product.txt"))
-hw_family = load_identifiers(File.join(bdir, "hw_family.txt"))
-hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
-svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
-svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
-
-missing_count = 0
-
-ARGV.each do |arg|
-  Dir.glob(arg).each do |file|
-    ndb = Recog::DB.new(file)
-    ndb.fingerprints.each do |f|
-      f.params.each do |k,v|
-        paramIndex, val = v
-        if ! fields[k]
-          puts "FIELD MISSING: #{k}"
-          missing_count += 1
-          fields[k] = true
-        end
-        next if paramIndex != 0
-        next if val.index("{") != nil
-        next if val.strip == ""
-        case k
-        when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
-          if ! vendors[val]
-            puts "VENDOR MISSING: #{val}"
-            missing_count += 1
-            vendors[val] = true
-          end
-        when "os.arch"
-          if ! os_arch[val]
-            puts "OS ARCH MISSING: #{val}"
-            missing_count += 1
-            os_arch[val] = true
-          end          
-        when "os.product"
-          if ! os_prod[val]
-            puts "OS PRODUCT MISSING: #{val}"
-            missing_count += 1
-            os_prod[val] = true
-          end
-        when "os.family"
-          if ! os_family[val]
-            puts "OS FAMILY MISSING: #{val}"
-            missing_count += 1
-            os_family[val] = true
-          end
-        when "os.device"
-          if ! os_device[val]
-            puts "OS DEVICE MISSING: #{val}"
-            missing_count += 1
-            os_device[val] = true
-          end
-        when "hw.product"
-          if ! hw_prod[val]
-            puts "HW PRODUCT MISSING: #{val}"
-            missing_count += 1
-            hw_prod[val] = true
-          end
-        when "hw.family"
-          if ! hw_family[val]
-            puts "HW FAMILY MISSING: #{val}"
-            missing_count += 1
-            hw_family[val] = true
-          end
-        when "hw.device"
-          if ! hw_device[val]
-            puts "HW DEVICE MISSING: #{val}"
-            missing_count += 1
-            hw_device[val] = true
-          end          
-        when "service.product", "service.component.product"
-          if ! svc_prod[val]
-            puts "SERVICE PRODUCT MISSING: #{val}"
-            missing_count += 1
-            svc_prod[val] = true
-          end            
-        when "service.family"
-          if ! svc_family[val]
-            puts "SERVICE FAMILY MISSING: #{val}"
-            missing_count += 1
-            svc_family[val] = true
-          end          
-        end
-      end
-    end
-  end
-end
-
-if options.write
-  # Write back the unique identifiers
-  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
-  write_identifiers(fields, File.join(bdir, "fields.txt"))
-  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
-  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
-  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
-  write_identifiers(os_device, File.join(bdir, "os_device.txt"))
-  write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
-  write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
-  write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
-  write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
-  write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
-end
-
-exit_code = (missing_count > 0 ? 1 : 0)
-exit(exit_code)

data/bin/recog_verify

@@ -1,63 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-require 'recog/verifier_factory'
-
-options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true)
-
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
-  opts.separator "Verifies that each fingerprint passes its internal tests."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-f", "--format FORMATTER",
-          "Choose a formatter.",
-          "  [s]ummary (default - failure/warning msgs and summary)",
-          "  [q]uiet (configured failure/warning msgs only)",
-          "  [d]etail  (fingerprint name with tests and expanded summary)") do |format|
-    if format.start_with? 'd'
-      options.detail = true
-    end
-    if format.start_with? 'q'
-      options.quiet = true
-    end
-  end
-
-  opts.on("-c", "--color", "Enable color in the output.") do
-    options.color = true
-  end
-
-  opts.on("--[no-]warnings", "Track warnings") do |o|
-    options.warnings = o
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.empty?
-  $stderr.puts 'Missing XML fingerprint files'
-  puts option_parser
-  exit(1)
-end
-
-warnings = 0
-failures = 0
-ARGV.each do |arg|
-  Dir.glob(arg).each do |file|
-    ndb = Recog::DB.new(file)
-    verifier = Recog::VerifierFactory.build(options, ndb)
-    verified = verifier.verify
-    failures += verifier.reporter.failure_count
-    warnings += verifier.reporter.warning_count
-  end
-end
-
-exit failures + warnings

data/cpe-remap.yaml

@@ -1,356 +0,0 @@
-mappings:
-  # The following section contains CPE application or 'a' remappings. These will
-  # ONLY be used for mapping Recog 'service' attributes.
-  a:
-    akamai:
-      products:
-        ghost: akamaighost
-    amazon:
-      products:
-        s3: amazon_simple_storage_service
-        cloudfront_load_balancer: amazon_cloudfront
-    apache:
-      products:
-        httpd: http_server
-    aprelium_technologies:
-      vendor: aprelium
-    alt-n:
-      vendor: altn
-    aruba_networks:
-      vendor: arubanetworks
-    bea:
-      products:
-        weblogic: weblogic_server
-    blue_coat:
-      vendor: bluecoat
-    carnegie_mellon_university:
-      vendor: cmu
-      products:
-        cyrus_imap: cyrus_imap_server
-    centos_webpanel:
-      vendor: centos-webpanel
-    check_point:
-      vendor: checkpoint
-    cherokee_project:
-      vendor: cherokee-project
-    cisco:
-      products:
-        apic: application_policy_infrastructure_controller
-    cloudflare:
-      products:
-        cloudflare_load_balancer: load_balancing
-    cpanel:
-      products:
-        cpanel_service_daemon: cpanel
-    crushftp:
-      products:
-        crushftp_web_interface: crushftp
-    cz.nic:
-      vendor: knot-dns
-    drupal:
-      products:
-        cms: drupal
-    embedthis:
-      products:
-        goahead_webserver: goahead
-    envoy_proxy:
-      vendor: envoyproxy
-    f5:
-      products:
-        big-ip: big-ip_local_traffic_manager
-        big-ip_ltm: big-ip_local_traffic_manager
-    fedora_project:
-      vendor: fedoraproject
-    google:
-      products:
-        google_web_services: web_server
-    ibm:
-      products:
-        lotus_domino: lotus_domino_server
-        ibm_domino: lotus_domino
-    ignite_realtime:
-      vendor: igniterealtime
-    intel:
-      products:
-        intel(r)_active_management_technology: active_management_technology
-        intel(r)_standard_manageability: standard_manageability
-    jamf:
-      products:
-        jamf_pro: jamf
-    kibana:
-      vendor: elasticsearch
-    kubernetes:
-      products:
-        nginx_ingress_controller: ingress-nginx
-    kodi:
-      products:
-        media_server: kodi
-    kong:
-      vendor: konghq
-      products:
-        gateway: kong_gateway
-    litespeed_technologies:
-      vendor: litespeedtech
-    lotus:
-      vendor: ibm
-    lynx_technology:
-      vendor: lynxtechnology
-      products:
-        twonky_media_server: twonky_server
-    mailenable:
-      products:
-        mail_server: mailenable
-    manageengine:
-      vendor: zohocorp
-      products:
-        adaudit_plus: manageengine_adaudit_plus
-        desktop_central: manageengine_desktop_central
-        opmanager: manageengine_opmanager
-    microsoft:
-      products:
-        active_directory_controller: active_directory
-        exchange_server_5.5: exchange_server
-        exchange_2000_server: exchange_server
-        exchange_2003_server: exchange_server
-        exchange_2007_server: exchange_server
-        lightweight_directory_server: active_directory_lightweight_directory_service
-        pws: personal_web_server
-    mod_ssl:
-      vendor: modssl
-    mod_wsgi:
-      vendor: modwsgi
-    # NIST took the vendor name from the website but apparently missed the `.in`
-    # in moinmo.in was part of the name
-    moinmoin:
-      vendor: moinmo
-    mort_bay:
-      vendor: mortbay
-    munin:
-      vendor: munin-monitoring
-    nlnet_labs:
-      vendor: nlnetlabs
-      products:
-        dnsd: name_server_daemon
-    net-snmp:
-      products:
-        snmp_agent: net-snmp
-    owncloud:
-      products:
-        owncloud_server: owncloud
-    parallels:
-      products:
-        plesk: parallels_plesk_panel
-    phoenix_contact:
-      vendor: phoenixcontact
-    plesk:
-      vendor: parallels
-    proftpd_project:
-      vendor: proftpd
-    progress:
-      products:
-        openedge_explorer: openedge
-    pulse_secure:
-      vendor: pulsesecure
-    realvnc_ltd.:
-      vendor: realvnc
-    red_hat:
-      vendor: redhat
-      products:
-        cygwin_x_server_project: cygwin
-        jboss_as: jboss_wildfly_application_server
-        jboss_eap: jboss_enterprise_application_platform
-        jbossweb: jboss_web_framework_kit
-        red_hat_directory_server: directory_server
-    rundeck:
-      vendor: pagerduty
-    serv-u:
-      vendor: solarwinds
-    squid_cache:
-      vendor: squid-cache
-    ssh_communications_security:
-      vendor: ssh
-      products:
-        ssh_tectia_server: tectia_server
-    standard_networks:
-      vendor: ipswitch
-    swagger:
-      vendor: smartbear
-    synology:
-      products:
-        dsm: diskstation_manager
-    tightvnc:
-      products:
-        desktop: tightvnc
-    tor_project:
-      vendor: torproject
-    traefik_labs:
-      vendor: traefik
-      products:
-        traefik_proxy: traefik
-    twistedmatrix:
-      products:
-        twisted_web: twistedweb
-    ubiquiti:
-      vendor: ui
-    vandyke_software:
-      vendor: vandyke
-    vmware:
-      products:
-        zimbra: zimbra_desktop
-        vcenter: vcenter_server
-    x.org:
-      products:
-        x.org_x11: x11
-
-  # The following section contains CPE operating system or 'o' remappings. These will
-  # ONLY be used for mapping Recog 'os' attributes.
-  o:
-    alpine:
-      vendor: alpinelinux
-      products:
-        linux: alpine_linux
-    apple:
-      products:
-        ios: iphone_os
-        mac_os: macos
-    brocade:
-      vendor: broadcom
-      products:
-        fabric_os: fabric_operating_system
-    centos:
-      products:
-        linux: centos
-    check_point:
-      vendor: checkpoint
-    cisco:
-      products:
-        adaptive_security_appliance: adaptive_security_appliance_software
-        nam: network_analysis_module_software
-        pix: pix_firewall_software
-        telepresence: telepresence_video_communication_server_software
-        vpn_3000_concentrator: vpn_3000_concentrator_series_software
-        wireless_lan_controller: wireless_lan_controller_software
-    citrix:
-      products:
-        netscaler: netscaler_firmware
-        netscaler_gateway: netscaler_gateway_firmware
-    cumulus:
-      vendor: cumulusnetworks
-    data_domain:
-      vendor: dell
-      products:
-        dd_os: emc_data_domain_os
-    debian:
-      products:
-        linux: debian_linux
-    hp:
-      products:
-        ilo: integrated_lights-out_firmware
-        ilo_firmware: integrated_lights-out_firmware
-        ilo_2: integrated_lights-out_2_firmware
-        ilo_3: integrated_lights-out_3_firmware
-        ilo_4: integrated_lights-out_4_firmware
-        ilo_5: integrated_lights-out_5_firmware
-        tru64_unix: tru64
-    ibm:
-      products:
-        os/400: os_400
-        i5/os: i5os
-    juniper:
-      products:
-        junos_os: junos
-    linux:
-      products:
-        linux: linux_kernel
-    microsoft:
-      products:
-        windows_server_2003_datacenter_edition: windows_server_2003
-        windows_server_2003_r2: windows_server_2003
-        windows_2008_r2: windows_server_2008
-        windows_server_2008_datacenter_edition: windows_server_2008
-        windows_server_2008_r2: windows_server_2008
-        windows_server_2008_r2_datacenter_edition: windows_server_2008
-        windows_server_2012_r2: windows_server_2012
-        nt: windows_nt
-        windows_nt_desktop: windows_nt
-        windows_nt_server: windows_nt
-        windows_server_2000: windows_2000
-        windows_2000_server: windows_2000
-        windows_2000_datacenter_server: windows_2000
-    oracle:
-      products:
-        ilom: integrated_lights_out_manager_firmware
-    palo_alto_networks:
-      vendor: paloaltonetworks
-    phoenix_contact:
-      vendor: phoenixcontact
-    red_hat:
-      vendor: redhat
-      products:
-        fedora_core_linux: fedora_core
-    software_house:
-      vendor: swhouse
-    sun:
-      products:
-        solaris: sunos
-    ubiquiti:
-      vendor: ui
-    ubuntu:
-      vendor: canonical
-      products:
-        linux: ubuntu_linux
-    vmware:
-      products:
-        photon_linux: photon_os
-        vmware_esx_server: esx
-        vmware_esxi_server: esxi
-    wind_river:
-      vendor: windriver
-
-  # The following section contains CPE hardware or 'h' remappings. These will
-  # ONLY be used for mapping Recog 'hw' attributes.
-  h:
-    apple:
-      products:
-        imac_(retina_4k_21.5-inch_2019): imac
-        imac_(retina_5k_27-inch_2017): imac
-        imac_(retina_5k_27-inch_2019): imac
-        imac_(retina_5k_27-inch_2020): imac
-        macbook_air_(13-inch_2017): macbook_air
-        macbook_air_(m1_2020): macbook_air
-        macbook_air_(retina_13-inch_2018): macbook_air
-        macbook_air_(retina_13-inch_2019): macbook_air
-        macbook_air_(retina_13-inch_2020): macbook_air
-        macbook_pro_(13-inch_2018_four_thunderbolt_3_ports): macbook_pro
-        macbook_pro_(13-inch_2019_two_thunderbolt_3_ports): macbook_pro
-        macbook_pro_(13-inch_2020): macbook_pro
-        macbook_pro_(13-inch_m1_2020): macbook_pro
-        macbook_pro_(15-inch_2018): macbook_pro
-        macbook_pro_(15-inch_2019): macbook_pro
-        macbook_pro_(16-inch_2019): macbook_pro
-        macbook_pro_(retina_13-inch_early_2015): macbook_pro
-        macbook_pro_(retina_15-inch_mid_2015): macbook_pro
-    cisco:
-      products:
-        nam: network_analysis_module
-    citrix:
-      products:
-        netscaler_sdx_gateway: netscaler_sdx
-    emc:
-      products:
-        celerra: celerra_network_attached_storage
-    hp:
-      products:
-        ilo: integrated_lights-out
-    kace:
-      vendor: dell
-      products:
-        k1000: kace_k1000_systems_management_appliance
-    phoenix_contact:
-      vendor: phoenixcontact
-    software_house:
-      vendor: swhouse
-    tandberg:
-      vendor: cisco
-    ubiquiti:
-      vendor: ui

data/features/data/failing_banners_fingerprints.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0"?>
-<fingerprints>
-  <fingerprint pattern="^=\(.\*.\)=-\.:\. \(\( Welcome to PureFTPd (\d+\..+) \)\) \.:\.-=\(.\*.\)=-$">
-    <example>=(&lt;*&gt;)=-.:. (( Welcome to PureFTPd 1.1.0 )) .:.-=(&lt;*&gt;)=-</example>
-    <description>Older Pure-FTPd versions</description>
-    <param pos="0" name="service.family" value="Pure-FTPd"/>
-    <param pos="0" name="service.product" value="Pure-FTPd"/>
-    <param pos="1" name="service.version"/>
-  </fingerprint>
-  <fingerprint pattern="^(\S+) FTP Server \(Solaris (\S+)\) ready\.?$" flags="REG_ICASE">
-    <description>SunOS/Solaris</description>
-    <example>example.com FTP server (Solaris 5.7) ready.</example>
-    <param pos="0" name="os.vendor" value="Sun"/>
-    <param pos="0" name="os.family" value="Solaris"/>
-    <param pos="0" name="os.product" value="Solaris"/>
-    <param pos="0" name="os.device" value="General"/>
-    <param pos="1" name="host.name"/>
-    <param pos="2" name="os.version"/>
-  </fingerprint>
-</fingerprints>

data/features/data/matching_banners_fingerprints.xml

@@ -1,23 +0,0 @@
-<?xml version="1.0"?>
-<fingerprints protocol="ftp" database_type="service">
-  <fingerprint pattern="^-{10} Welcome to Pure-FTPd (.*)-{10}$">
-    <example>---------- Welcome to Pure-FTPd ----------</example>
-    <description>Pure-FTPd
-      Config data can be zero or more of: [privsep] [TLS]
-      </description>
-    <param pos="1" name="pureftpd.config"/>
-    <param pos="0" name="service.family" value="Pure-FTPd"/>
-    <param pos="0" name="service.product" value="Pure-FTPd"/>
-    <param pos="0" name="service.protocol" value="ftp"/>
-  </fingerprint>
-  <fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
-    <description>SunOS/Solaris</description>
-    <example>example.com FTP server (SunOS 5.7) ready.</example>
-    <param pos="0" name="os.vendor" value="Sun"/>
-    <param pos="0" name="os.family" value="Solaris"/>
-    <param pos="0" name="os.product" value="Solaris"/>
-    <param pos="0" name="os.device" value="General"/>
-    <param pos="1" name="host.name"/>
-    <param pos="2" name="os.version"/>
-  </fingerprint>
-</fingerprints>

data/features/data/multiple_banners_fingerprints.xml

@@ -1,32 +0,0 @@
-<?xml version="1.0"?>
-<fingerprints>
-  <fingerprint pattern="FTP">
-    <example>---- FTP Stuff ----</example>
-    <example>FTP server</example>
-    <description>Generic FTP,
-      Checks for the existence of the word FTP in the line
-    </description>
-    <!-- Asserting nothing -->
-  </fingerprint>
-  <fingerprint pattern="^-{10} Welcome to Pure-FTPd (.*)-{10}$">
-    <example>---------- Welcome to Pure-FTPd ----------</example>
-    <description>Pure-FTPd
-      Config data can be zero or more of: [privsep] [TLS]
-    </description>
-    <param pos="1" name="pureftpd.config"/>
-    <param pos="0" name="service.family" value="Pure-FTPd"/>
-    <param pos="0" name="service.product" value="Pure-FTPd"/>
-    <param pos="0" name="service.protocol" value="ftp"/>
-  </fingerprint>
-  <fingerprint pattern="^(\S+) FTP Server \(SunOS (\S+)\) ready\.?$" flags="REG_ICASE">
-    <description>SunOS/Solaris</description>
-    <example>example.com FTP server (SunOS 5.7) ready.</example>
-    <param pos="0" name="service.protocol" value="ftp"/>
-    <param pos="0" name="os.vendor" value="Sun"/>
-    <param pos="0" name="os.family" value="Solaris"/>
-    <param pos="0" name="os.product" value="Solaris"/>
-    <param pos="0" name="os.device" value="General"/>
-    <param pos="1" name="host.name"/>
-    <param pos="2" name="os.version"/>
-  </fingerprint>
-</fingerprints>

data/features/data/no_tests.xml

@@ -1,3 +0,0 @@
-<?xml version="1.0"?>
-<fingerprints>
-</fingerprints>

data/features/data/sample_banner.txt

@@ -1,2 +0,0 @@
----------- Welcome to Pure-FTPd [privsep] [TLS] ----------
-polaris FTP server (SunOS 5.8) ready.

data/features/data/successful_tests.xml

@@ -1,18 +0,0 @@
-<?xml version="1.0"?>
-<fingerprints>
-   <fingerprint pattern="^Cisco-SIPGateway/IOS-([\d\.x]+)$">
-      <description>Cisco SIPGateway</description>
-      <example os.version="12.x">Cisco-SIPGateway/IOS-12.x</example>
-      <param pos="0" name="os.vendor" value="Cisco"/>
-      <param pos="0" name="os.product" value="IOS"/>
-      <param pos="1" name="os.version"/>
-   </fingerprint>
-   <fingerprint pattern="^bar ([\d.]+)$">
-     <description>bar test</description>
-     <example os.version="1.0" >bar 1.0</example>
-     <example os.version="2.0" >bar 2.0</example>
-     <example os.version="2.1" >bar 2.1</example>
-     <param pos="1" name="os.version" />
-     <param pos="0" name="os.name" value="Bar" />
-   </fingerprint>
-</fingerprints>