recog 2.3.23 → 3.0.1

data/bin/recog_export

@@ -1,81 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-def squash_lines(str)
-  str.split(/\n/).join(' ').gsub(/\s+/, ' ')
-end
-
-def export_text(options)
-end
-
-def export_ruby(options)
-  $stdout.puts "# Recog fingerprint database export [ #{File.basename(options.xml_file)} ] on #{Time.now.to_s}"
-  $stdout.puts "fp_str   = '' # Set this value to the match string"
-  $stdout.puts "fp_match = {} # Match results are stored here"
-  $stdout.puts ""
-  $stdout.puts "case fp_str"
-  options.db.fingerprints.each do |fp|
-    puts "  # #{squash_lines fp.name}"
-    puts "  when /#{fp.regex.to_s}/"
-    fp.tests.each do |test|
-      puts "    # Example: #{squash_lines test}"
-    end
-    fp.params.each_pair do |k,v|
-      if v[0] == 0
-        puts "    fp_match[#{k.inspect}] = #{v[1].inspect}"
-      else
-        puts "    fp_match[#{k.inspect}] = $#{v[0].to_s}"
-      end
-    end
-    puts ""
-  end
-  $stdout.puts "end"
-end
-
-
-options = OpenStruct.new(etype: :ruby)
-
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINTS_FILE"
-  opts.separator "Exports an XML fingerprint database to another format."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-t", "--type type", 
-          "Choose a type of export.",
-          "  [r]uby (default - export a ruby case statement with regular expressions)",
-          "  [t]ext (export a text description of the fingerprints)") do |etype|
-    case etype.downcase
-    when /^r/
-      options.etype = :ruby
-    when /^t/
-      options.etype = :text
-    end
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.count != 1
-  puts option_parser
-  exit
-end
-
-options.xml_file = ARGV.shift
-options.db = Recog::DB.new(options.xml_file)
-
-case options.etype
-when :ruby
-  export_ruby(options)
-when :text
-  export_text(options)
-end
-

data/bin/recog_standardize

@@ -1,163 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'optparse'
-require 'ostruct'
-require 'recog'
-
-def load_identifiers(path)
-  res = {}
-  File.readlines(path).map{|line| line.strip}.each do |ident|
-    res[ident] = true
-  end
-  return res
-end
-
-def write_identifiers(vals, path)
-  res = []
-  vals.each_pair do |k,v|
-    res = res.push(k)
-  end
-  res = res.map{|x| x.strip}.select{|x| x.length > 0}.sort.uniq
-  File.write(path, res.join("\n") + "\n")
-end
-
-bdir = File.expand_path(File.join(File.dirname(__FILE__), "..", "identifiers"))
-
-options = OpenStruct.new(write: false)
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
-  opts.separator "Verifies that each fingerprint asserts known identifiers."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-w", "--write") do
-      options.write = true
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.empty?
-  $stderr.puts 'Missing XML fingerprint files'
-  puts option_parser
-  exit(1)
-end
-
-# Load the unique identifiers
-vendors = load_identifiers(File.join(bdir, "vendor.txt"))
-fields = load_identifiers(File.join(bdir, "fields.txt"))
-os_arch = load_identifiers(File.join(bdir, "os_architecture.txt"))
-os_prod = load_identifiers(File.join(bdir, "os_product.txt"))
-os_family = load_identifiers(File.join(bdir, "os_family.txt"))
-os_device = load_identifiers(File.join(bdir, "os_device.txt"))
-hw_prod = load_identifiers(File.join(bdir, "hw_product.txt"))
-hw_family = load_identifiers(File.join(bdir, "hw_family.txt"))
-hw_device = load_identifiers(File.join(bdir, "hw_device.txt"))
-svc_prod = load_identifiers(File.join(bdir, "service_product.txt"))
-svc_family = load_identifiers(File.join(bdir, "service_family.txt"))
-
-missing_count = 0
-
-ARGV.each do |arg|
-  Dir.glob(arg).each do |file|
-    ndb = Recog::DB.new(file)
-    ndb.fingerprints.each do |f|
-      f.params.each do |k,v|
-        paramIndex, val = v
-        if ! fields[k]
-          puts "FIELD MISSING: #{k}"
-          missing_count += 1
-          fields[k] = true
-        end
-        next if paramIndex != 0
-        next if val.index("{") != nil
-        next if val.strip == ""
-        case k
-        when "os.vendor", "service.vendor", "service.component.vendor", "hw.vendor"
-          if ! vendors[val]
-            puts "VENDOR MISSING: #{val}"
-            missing_count += 1
-            vendors[val] = true
-          end
-        when "os.arch"
-          if ! os_arch[val]
-            puts "OS ARCH MISSING: #{val}"
-            missing_count += 1
-            os_arch[val] = true
-          end          
-        when "os.product"
-          if ! os_prod[val]
-            puts "OS PRODUCT MISSING: #{val}"
-            missing_count += 1
-            os_prod[val] = true
-          end
-        when "os.family"
-          if ! os_family[val]
-            puts "OS FAMILY MISSING: #{val}"
-            missing_count += 1
-            os_family[val] = true
-          end
-        when "os.device"
-          if ! os_device[val]
-            puts "OS DEVICE MISSING: #{val}"
-            missing_count += 1
-            os_device[val] = true
-          end
-        when "hw.product"
-          if ! hw_prod[val]
-            puts "HW PRODUCT MISSING: #{val}"
-            missing_count += 1
-            hw_prod[val] = true
-          end
-        when "hw.family"
-          if ! hw_family[val]
-            puts "HW FAMILY MISSING: #{val}"
-            missing_count += 1
-            hw_family[val] = true
-          end
-        when "hw.device"
-          if ! hw_device[val]
-            puts "HW DEVICE MISSING: #{val}"
-            missing_count += 1
-            hw_device[val] = true
-          end          
-        when "service.product", "service.component.product"
-          if ! svc_prod[val]
-            puts "SERVICE PRODUCT MISSING: #{val}"
-            missing_count += 1
-            svc_prod[val] = true
-          end            
-        when "service.family"
-          if ! svc_family[val]
-            puts "SERVICE FAMILY MISSING: #{val}"
-            missing_count += 1
-            svc_family[val] = true
-          end          
-        end
-      end
-    end
-  end
-end
-
-if options.write
-  # Write back the unique identifiers
-  write_identifiers(vendors, File.join(bdir, "vendor.txt"))
-  write_identifiers(fields, File.join(bdir, "fields.txt"))
-  write_identifiers(os_arch, File.join(bdir, "os_architecture.txt"))
-  write_identifiers(os_prod, File.join(bdir, "os_product.txt"))
-  write_identifiers(os_family, File.join(bdir, "os_family.txt"))
-  write_identifiers(os_device, File.join(bdir, "os_device.txt"))
-  write_identifiers(hw_prod, File.join(bdir, "hw_product.txt"))
-  write_identifiers(hw_family, File.join(bdir, "hw_family.txt"))
-  write_identifiers(hw_device, File.join(bdir, "hw_device.txt"))
-  write_identifiers(svc_prod, File.join(bdir, "service_product.txt"))
-  write_identifiers(svc_family, File.join(bdir, "service_family.txt"))
-end
-
-exit_code = (missing_count > 0 ? 1 : 0)
-exit(exit_code)

data/bin/recog_verify

@@ -1,98 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
-require 'nokogiri'
-require 'optparse'
-require 'ostruct'
-require 'recog'
-require 'recog/formatter'
-require 'recog/verifier'
-require 'recog/verify_reporter'
-
-options = OpenStruct.new(color: false, detail: false, quiet: false, warnings: true, schema: nil)
-
-option_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{$0} [options] XML_FINGERPRINT_FILE1 ..."
-  opts.separator "Verifies that each fingerprint passes its internal tests."
-  opts.separator ""
-  opts.separator "Options"
-
-  opts.on("-f", "--format FORMATTER",
-          "Choose a formatter.",
-          "  [s]ummary (default - failure/warning msgs and summary)",
-          "  [q]uiet (configured failure/warning msgs only)",
-          "  [d]etail  (fingerprint name with tests and expanded summary)") do |format|
-    if format.start_with? 'd'
-      options.detail = true
-    end
-    if format.start_with? 'q'
-      options.quiet = true
-    end
-  end
-
-  opts.on("-c", "--color", "Enable color in the output.") do
-    options.color = true
-  end
-
-  opts.on("--[no-]warnings", "Track warnings") do |o|
-    options.warnings = o
-  end
-
-  opts.on("--schema-location SCHEMA_FILE", "Location of the Recog XSD file. If not specified, validation will not be run.") do |schema_file|
-    options.schema = Nokogiri::XML::Schema(File.read(schema_file))
-  end
-
-  opts.on("-h", "--help", "Show this message.") do
-    puts opts
-    exit
-  end
-end
-option_parser.parse!(ARGV)
-
-if ARGV.empty?
-  $stderr.puts 'Missing XML fingerprint files'
-  puts option_parser
-  exit(1)
-end
-
-warnings = 0
-failures = 0
-formatter = Recog::Formatter.new(options, $stdout)
-ARGV.each do |arg|
-  Dir.glob(arg).each do |file|
-    # Create a new reporter per XML file to hold context on success/warn/fails
-    reporter  = Recog::VerifyReporter.new(options, formatter, file)
-
-    begin
-      # Validate the XML database against the recog schema first, if requested
-      if options.schema
-        errors = options.schema.validate(Nokogiri::XML(File.read(file)))
-        if errors.size > 0
-          reporter.report(0) do
-            errors.each do |error|
-              reporter.failure(error.message, error.line)
-            end
-          end
-          # Skip validation of individual fingerprints since the XML itself
-          # is likely malformed.
-          next
-        end
-      end
-
-      # Now read the XML file directly and validate the fingerprints
-      # themselves
-      db = Recog::DB.new(file)
-      verifier = Recog::Verifier.new(db, reporter)
-      verifier.verify
-    rescue Recog::FingerprintParseError => e
-      reporter.failure(e.message, e.line_number)
-    rescue => e
-      reporter.failure(e.message)
-    ensure
-      failures += reporter.failure_count
-      warnings += reporter.warning_count
-    end
-  end
-end
-
-exit failures + warnings

data/cpe-remap.yaml

@@ -1,374 +0,0 @@
-mappings:
-  # The following section contains CPE application or 'a' remappings. These will
-  # ONLY be used for mapping Recog 'service' attributes.
-  a:
-    akamai:
-      products:
-        ghost: akamaighost
-    amazon:
-      products:
-        s3: amazon_simple_storage_service
-        cloudfront_load_balancer: amazon_cloudfront
-    apache:
-      products:
-        httpd: http_server
-    aprelium_technologies:
-      vendor: aprelium
-    alt-n:
-      vendor: altn
-    aruba_networks:
-      vendor: arubanetworks
-    atlassian:
-      products:
-        confluence: confluence_server
-    bea:
-      products:
-        weblogic: weblogic_server
-    blue_coat:
-      vendor: bluecoat
-    carnegie_mellon_university:
-      vendor: cmu
-      products:
-        cyrus_imap: cyrus_imap_server
-    centos_webpanel:
-      vendor: centos-webpanel
-    check_point:
-      vendor: checkpoint
-    cherokee_project:
-      vendor: cherokee-project
-    cisco:
-      products:
-        apic: application_policy_infrastructure_controller
-    cloudflare:
-      products:
-        cloudflare_load_balancer: load_balancing
-    cpanel:
-      products:
-        cpanel_service_daemon: cpanel
-    crushftp:
-      products:
-        crushftp_web_interface: crushftp
-    cz.nic:
-      vendor: knot-dns
-    drupal:
-      products:
-        cms: drupal
-    embedthis:
-      products:
-        goahead_webserver: goahead
-    envoy_proxy:
-      vendor: envoyproxy
-    f5:
-      products:
-        big-ip: big-ip_local_traffic_manager
-        big-ip_ltm: big-ip_local_traffic_manager
-    fedora_project:
-      vendor: fedoraproject
-    google:
-      products:
-        google_web_services: web_server
-    ibm:
-      products:
-        lotus_domino: lotus_domino_server
-        ibm_domino: lotus_domino
-    ignite_realtime:
-      vendor: igniterealtime
-    intel:
-      products:
-        intel(r)_active_management_technology: active_management_technology
-        intel(r)_standard_manageability: standard_manageability
-    jamf:
-      products:
-        jamf_pro: jamf
-    kibana:
-      vendor: elasticsearch
-    kubernetes:
-      products:
-        nginx_ingress_controller: ingress-nginx
-    kodi:
-      products:
-        media_server: kodi
-    kong:
-      vendor: konghq
-      products:
-        gateway: kong_gateway
-    litespeed_technologies:
-      vendor: litespeedtech
-    lotus:
-      vendor: ibm
-    lynx_technology:
-      vendor: lynxtechnology
-      products:
-        twonky_media_server: twonky_server
-    mailenable:
-      products:
-        mail_server: mailenable
-    manageengine:
-      vendor: zohocorp
-      products:
-        adaudit_plus: manageengine_adaudit_plus
-        desktop_central: manageengine_desktop_central
-        opmanager: manageengine_opmanager
-    microsoft:
-      products:
-        active_directory_controller: active_directory
-        exchange_server_5.5: exchange_server
-        exchange_2000_server: exchange_server
-        exchange_2003_server: exchange_server
-        exchange_2007_server: exchange_server
-        lightweight_directory_server: active_directory_lightweight_directory_service
-        pws: personal_web_server
-    mod_ssl:
-      vendor: modssl
-    mod_wsgi:
-      vendor: modwsgi
-    # NIST took the vendor name from the website but apparently missed the `.in`
-    # in moinmo.in was part of the name
-    moinmoin:
-      vendor: moinmo
-    mort_bay:
-      vendor: mortbay
-    munin:
-      vendor: munin-monitoring
-    nginx:
-      vendor: f5
-    nlnet_labs:
-      vendor: nlnetlabs
-      products:
-        dnsd: name_server_daemon
-    net-snmp:
-      products:
-        snmp_agent: net-snmp
-    owncloud:
-      products:
-        owncloud_server: owncloud
-    parallels:
-      products:
-        plesk: parallels_plesk_panel
-    phoenix_contact:
-      vendor: phoenixcontact
-    plesk:
-      vendor: parallels
-    proftpd_project:
-      vendor: proftpd
-    progress:
-      products:
-        openedge_explorer: openedge
-    pulse_secure:
-      vendor: pulsesecure
-    realvnc_ltd.:
-      vendor: realvnc
-    red_hat:
-      vendor: redhat
-      products:
-        cygwin_x_server_project: cygwin
-        jboss_as: jboss_wildfly_application_server
-        jboss_eap: jboss_enterprise_application_platform
-        jbossweb: jboss_web_framework_kit
-        red_hat_directory_server: directory_server
-    rundeck:
-      vendor: pagerduty
-    serv-u:
-      vendor: solarwinds
-    squid_cache:
-      vendor: squid-cache
-    ssh_communications_security:
-      vendor: ssh
-      products:
-        ssh_tectia_server: tectia_server
-    standard_networks:
-      vendor: ipswitch
-    swagger:
-      vendor: smartbear
-    synology:
-      products:
-        dsm: diskstation_manager
-    tightvnc:
-      products:
-        desktop: tightvnc
-    tor_project:
-      vendor: torproject
-    traefik_labs:
-      vendor: traefik
-      products:
-        traefik_proxy: traefik
-    twistedmatrix:
-      products:
-        twisted_web: twistedweb
-    ubiquiti:
-      vendor: ui
-    vandyke_software:
-      vendor: vandyke
-    vmware:
-      products:
-        zimbra: zimbra_desktop
-        vcenter: vcenter_server
-    x.org:
-      products:
-        x.org_x11: x11
-    xiongmai_technology:
-      vendor: xiongmaitech
-    zaphoyd_studios:
-      vendor: zaphoyd
-      products:
-        websocket++: websocketpp
-
-  # The following section contains CPE operating system or 'o' remappings. These will
-  # ONLY be used for mapping Recog 'os' attributes.
-  o:
-    alpine:
-      vendor: alpinelinux
-      products:
-        linux: alpine_linux
-    apple:
-      products:
-        ios: iphone_os
-        mac_os: macos
-    brocade:
-      vendor: broadcom
-      products:
-        fabric_os: fabric_operating_system
-    centos:
-      products:
-        linux: centos
-    check_point:
-      vendor: checkpoint
-    cisco:
-      products:
-        adaptive_security_appliance: adaptive_security_appliance_software
-        mds_9000: mds_9000_san-os
-        nam: network_analysis_module_software
-        pix: pix_firewall_software
-        telepresence: telepresence_video_communication_server_software
-        vpn_3000_concentrator: vpn_3000_concentrator_series_software
-        wireless_lan_controller: wireless_lan_controller_software
-    citrix:
-      products:
-        netscaler: netscaler_firmware
-        netscaler_gateway: netscaler_gateway_firmware
-    cumulus:
-      vendor: cumulusnetworks
-    data_domain:
-      vendor: dell
-      products:
-        dd_os: emc_data_domain_os
-    debian:
-      products:
-        linux: debian_linux
-    eltex:
-      vendor: eltex-co
-    fedora_project:
-      vendor: fedoraproject
-    hp:
-      products:
-        ilo: integrated_lights-out_firmware
-        ilo_firmware: integrated_lights-out_firmware
-        ilo_2: integrated_lights-out_2_firmware
-        ilo_3: integrated_lights-out_3_firmware
-        ilo_4: integrated_lights-out_4_firmware
-        ilo_5: integrated_lights-out_5_firmware
-        tru64_unix: tru64
-    ibm:
-      products:
-        os/400: os_400
-        i5/os: i5os
-    juniper:
-      products:
-        junos_os: junos
-    linux:
-      products:
-        linux: linux_kernel
-    microsoft:
-      products:
-        windows_server_2003_datacenter_edition: windows_server_2003
-        windows_server_2003_r2: windows_server_2003
-        windows_2008_r2: windows_server_2008
-        windows_server_2008_datacenter_edition: windows_server_2008
-        windows_server_2008_r2: windows_server_2008
-        windows_server_2008_r2_datacenter_edition: windows_server_2008
-        windows_server_2012_r2: windows_server_2012
-        nt: windows_nt
-        windows_nt_desktop: windows_nt
-        windows_nt_server: windows_nt
-        windows_server_2000: windows_2000
-        windows_2000_server: windows_2000
-        windows_2000_datacenter_server: windows_2000
-    oracle:
-      products:
-        ilom: integrated_lights_out_manager_firmware
-    palo_alto_networks:
-      vendor: paloaltonetworks
-    phoenix_contact:
-      vendor: phoenixcontact
-    red_hat:
-      vendor: redhat
-    software_house:
-      vendor: swhouse
-    sun:
-      products:
-        solaris: sunos
-    ubiquiti:
-      vendor: ui
-    ubuntu:
-      vendor: canonical
-      products:
-        linux: ubuntu_linux
-    vmware:
-      products:
-        photon_linux: photon_os
-        vmware_esx_server: esx
-        vmware_esxi_server: esxi
-    wind_river:
-      vendor: windriver
-    xiongmai_technology:
-      vendor: xiongmaitech
-
-  # The following section contains CPE hardware or 'h' remappings. These will
-  # ONLY be used for mapping Recog 'hw' attributes.
-  h:
-    apple:
-      products:
-        imac_(retina_4k_21.5-inch_2019): imac
-        imac_(retina_5k_27-inch_2017): imac
-        imac_(retina_5k_27-inch_2019): imac
-        imac_(retina_5k_27-inch_2020): imac
-        macbook_air_(13-inch_2017): macbook_air
-        macbook_air_(m1_2020): macbook_air
-        macbook_air_(retina_13-inch_2018): macbook_air
-        macbook_air_(retina_13-inch_2019): macbook_air
-        macbook_air_(retina_13-inch_2020): macbook_air
-        macbook_pro_(13-inch_2018_four_thunderbolt_3_ports): macbook_pro
-        macbook_pro_(13-inch_2019_two_thunderbolt_3_ports): macbook_pro
-        macbook_pro_(13-inch_2020): macbook_pro
-        macbook_pro_(13-inch_m1_2020): macbook_pro
-        macbook_pro_(15-inch_2018): macbook_pro
-        macbook_pro_(15-inch_2019): macbook_pro
-        macbook_pro_(16-inch_2019): macbook_pro
-        macbook_pro_(retina_13-inch_early_2015): macbook_pro
-        macbook_pro_(retina_15-inch_mid_2015): macbook_pro
-    cisco:
-      products:
-        nam: network_analysis_module
-    citrix:
-      products:
-        netscaler_sdx_gateway: netscaler_sdx
-    eltex:
-      vendor: eltex-co
-    emc:
-      products:
-        celerra: celerra_network_attached_storage
-    hp:
-      products:
-        ilo: integrated_lights-out
-    kace:
-      vendor: dell
-      products:
-        k1000: kace_k1000_systems_management_appliance
-    phoenix_contact:
-      vendor: phoenixcontact
-    software_house:
-      vendor: swhouse
-    tandberg:
-      vendor: cisco
-    ubiquiti:
-      vendor: ui